particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test/api.sh: randomize used GCP zone from the region

Browse source 
The `api.sh` test currently always defaults to "<REGION>-a" zone when
creating instance using the built image. The resources in a zone may get
exhausted and the solution is to use a different zone. Currently even a
CI job retry won't help with mitigation of such error during a CI run.

Modify `api.sh` to pick random GCP zone for a given region when creating
a compute instance. Use only GCP zones which are "UP".

The `cloud-cleaner` relied on the behavior of `api.sh` to always choose
the "<REGION>-a" zone. Guessing the chosen zone in `cloud-cleaner` is
not viable, but thankfully the instance name is by default unique for
the whole GCP project. Modify `cloud-cleaner` to iterate over all
available zones in the used region and try to delete the specific
instance in each of them.

Make `ComputeZonesInRegion` method from the `internal/cloud/gcp` package
exported and use it in `cloud-cleaner` for getting the list of available
zones in a region.

Signed-off-by: Tomas Hozza <thozza@redhat.com>
This commit is contained in:
Tomas Hozza 2021-06-22 13:27:45 +02:00 • committed by Ondřej Budai
parent 1fd151ce28
commit 3a0540dff0
3 changed files with 37 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

34
cmd/cloud-cleaner/main.go
View file

@ -27,8 +27,6 @@ func cleanupGCP(testID string, wg *sync.WaitGroup) {
log.Println("[GCP] Error: 'GCP_REGION' is not set in the environment.")
return
}
// api.sh test uses '--zone="$GCP_REGION-a"'
GCPZone := fmt.Sprintf("%s-a", GCPRegion)
GCPBucket, ok := os.LookupEnv("GCP_BUCKET")
if !ok {
log.Println("[GCP] Error: 'GCP_BUCKET' is not set in the environment.")
@ -63,16 +61,31 @@ func cleanupGCP(testID string, wg *sync.WaitGroup) {
ctx := context.Background()
// Try to delete potentially running instance
log.Printf("[GCP] 🧹 Deleting VM instance %s in %s. "+
"This should fail if the test succedded.", GCPInstance, GCPZone)
err = g.ComputeInstanceDelete(ctx, GCPZone, GCPInstance)
// api.sh chooses a random GCP Zone from the set Region. Since we
// don't know which one it is, iterate over all Zones in the Region
// and try to delete the instance. Unless the instance has set
// "VmDnsSetting:ZonalOnly", which we don't do, this is safe and the
// instance name must be unique for the whole GCP project.
GCPZones, err := g.ComputeZonesInRegion(ctx, GCPRegion)
if err != nil {
log.Printf("[GCP] Error: %v", err)
log.Printf("[GCP] Error: Failed to get available Zones for the '%s' Region: %v", GCPRegion, err)
return
}
for _, GCPZone := range GCPZones {
log.Printf("[GCP] 🧹 Deleting VM instance %s in %s. "+
"This should fail if the test succeeded.", GCPInstance, GCPZone)
err = g.ComputeInstanceDelete(ctx, GCPZone, GCPInstance)
if err == nil {
// If an instance with the given name was successfully deleted in one of the Zones, we are done.
break
} else {
log.Printf("[GCP] Error: %v", err)
}
}
// Try to clean up storage of cache objects after image import job
log.Println("[GCP] 🧹 Cleaning up cache objects from storage after image " +
"import. This should fail if the test succedded.")
"import. This should fail if the test succeeded.")
cacheObjects, errs := g.StorageImageImportCleanup(ctx, GCPImage)
for _, err = range errs {
log.Printf("[GCP] Error: %v", err)
@ -94,7 +107,7 @@ func cleanupGCP(testID string, wg *sync.WaitGroup) {
}
// Try to delete the imported image
log.Printf("[GCP] 🧹 Deleting image %s. This should fail if the test succedded.", GCPImage)
log.Printf("[GCP] 🧹 Deleting image %s. This should fail if the test succeeded.", GCPImage)
err = g.ComputeImageDelete(ctx, GCPImage)
if err != nil {
log.Printf("[GCP] Error: %v", err)
@ -119,14 +132,14 @@ func cleanupAzure(testID string, wg *sync.WaitGroup) {
// Delete the vhd image
imageName := "image-" + testID + ".vhd"
log.Println("[Azure] Deleting image. This should fail if the test succedded.")
log.Println("[Azure] Deleting image. This should fail if the test succeeded.")
err = azuretest.DeleteImageFromAzure(creds, imageName)
if err != nil {
log.Printf("[Azure] Error: %v", err)
}
// Delete all remaining resources (see the full list in the CleanUpBootedVM function)
log.Println("[Azure] Cleaning up booted VM. This should fail if the test succedded.")
log.Println("[Azure] Cleaning up booted VM. This should fail if the test succeeded.")
parameters := azuretest.NewDeploymentParameters(creds, imageName, testID, "")
clientCredentialsConfig := auth.NewClientCredentialsConfig(creds.ClientID, creds.ClientSecret, creds.TenantID)
authorizer, err := clientCredentialsConfig.Authorizer()
@ -149,6 +162,7 @@ func main() {
if err != nil {
log.Fatalf("Failed to get testID: %v", err)
}
log.Printf("TEST_ID=%s", testID)
var wg sync.WaitGroup
wg.Add(2)