From 1017aee4383ee8f4d95f2eaabdb3ff922f092b06 Mon Sep 17 00:00:00 2001 From: Tomas Hozza Date: Mon, 16 May 2022 14:12:57 +0200 Subject: [PATCH] cloud-cleaner: clean up GCE instances in all regions and zones Since the `api.sh` test case is using random GCE zone from a random GCE region which name starts with the `GCP_REGION` CI environment variable. Since the used region name is not known to the `cloud-cleaner`, it has to iterate over all potential GCE regions and their zones. We can not simply filter the VM instance name a list of instances, because any `instances` API call requires a zone name to be provided. Add a new internal `cloud/gcp` package method to list existing GCE regions based on a provided filter. --- cmd/cloud-cleaner/main.go | 40 ++++++++++++---------- internal/cloud/gcp/compute.go | 64 +++++++++++++++++++++++------------ 2 files changed, 65 insertions(+), 39 deletions(-) diff --git a/cmd/cloud-cleaner/main.go b/cmd/cloud-cleaner/main.go index 1bb05d282..b87245d82 100644 --- a/cmd/cloud-cleaner/main.go +++ b/cmd/cloud-cleaner/main.go @@ -22,7 +22,7 @@ func cleanupGCP(testID string, wg *sync.WaitGroup) { log.Println("[GCP] Running clean up") - GCPRegion, ok := os.LookupEnv("GCP_REGION") + GCPRegionName, ok := os.LookupEnv("GCP_REGION") if !ok { log.Println("[GCP] Error: 'GCP_REGION' is not set in the environment.") return @@ -61,25 +61,31 @@ func cleanupGCP(testID string, wg *sync.WaitGroup) { ctx := context.Background() // Try to delete potentially running instance - // api.sh chooses a random GCP Zone from the set Region. Since we - // don't know which one it is, iterate over all Zones in the Region - // and try to delete the instance. Unless the instance has set - // "VmDnsSetting:ZonalOnly", which we don't do, this is safe and the - // instance name must be unique for the whole GCP project. - GCPZones, err := g.ComputeZonesInRegion(ctx, GCPRegion) + // api.sh chooses a random GCP Zone from regions which names start with the + // `GCPRegionName` value. Since we don't know which one it is, iterate over + // all Regions and Zones in them and try to delete the instance. Unless the + // instance has set "VmDnsSetting:ZonalOnly", which we don't do, this is + // safe and the instance name must be unique for the whole GCP project. + GCPRegions, err := g.ComputeRegionsList(ctx, fmt.Sprintf("name:%s* AND status=UP", GCPRegionName)) if err != nil { - log.Printf("[GCP] Error: Failed to get available Zones for the '%s' Region: %v", GCPRegion, err) - return + log.Printf("[GCP] Error: Failed to list GCE regions starting with %q: %v", GCPRegionName, err) } - for _, GCPZone := range GCPZones { - log.Printf("[GCP] 🧹 Deleting VM instance %s in %s. "+ - "This should fail if the test succeeded.", GCPInstance, GCPZone) - err = g.ComputeInstanceDelete(ctx, GCPZone, GCPInstance) - if err == nil { - // If an instance with the given name was successfully deleted in one of the Zones, we are done. + for _, GCPRegion := range GCPRegions { + GCPZones, err := g.ComputeUpZonesInRegion(ctx, GCPRegion) + if err != nil { + log.Printf("[GCP] Error: Failed to get available Zones for the '%s' Region: %v", GCPRegionName, err) break - } else { - log.Printf("[GCP] Error: %v", err) + } + for _, GCPZone := range GCPZones { + log.Printf("[GCP] 🧹 Deleting VM instance %s in %s. "+ + "This should fail if the test succeeded.", GCPInstance, GCPZone.GetName()) + err = g.ComputeInstanceDelete(ctx, GCPZone.GetName(), GCPInstance) + if err == nil { + // If an instance with the given name was successfully deleted in one of the Zones, we are done. + break + } else { + log.Printf("[GCP] Error: %v", err) + } } } diff --git a/internal/cloud/gcp/compute.go b/internal/cloud/gcp/compute.go index 1c5e4be68..77aeb0ea8 100644 --- a/internal/cloud/gcp/compute.go +++ b/internal/cloud/gcp/compute.go @@ -7,6 +7,7 @@ import ( compute "cloud.google.com/go/compute/apiv1" "github.com/osbuild/osbuild-composer/internal/common" + "google.golang.org/api/iterator" "google.golang.org/api/option" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" ) @@ -335,35 +336,20 @@ func (g *GCP) ComputeDiskDelete(ctx context.Context, zone, disk string) error { return err } -// ComputeZonesInRegion returns list of zones within the given GCE Region, which are "UP". +// ComputeUpZonesInRegion returns list of zones within the given GCE Region, which are "UP". // // Uses: // - Compute Engine API -func (g *GCP) ComputeZonesInRegion(ctx context.Context, region string) ([]string, error) { - var zones []string +func (g *GCP) ComputeUpZonesInRegion(ctx context.Context, region *computepb.Region) ([]*computepb.Zone, error) { + var zones []*computepb.Zone - regionsClient, err := compute.NewRegionsRESTClient(ctx, option.WithCredentials(g.creds)) - if err != nil { - return nil, fmt.Errorf("failed to get Compute Engine Regions client: %v", err) - } - defer regionsClient.Close() zonesClient, err := compute.NewZonesRESTClient(ctx, option.WithCredentials(g.creds)) if err != nil { return nil, fmt.Errorf("failed to get Compute Engine Zones client: %v", err) } defer zonesClient.Close() - // Get available zones in the given region - getRegionReq := &computepb.GetRegionRequest{ - Project: g.GetProjectID(), - Region: region, - } - regionObj, err := regionsClient.Get(ctx, getRegionReq) - if err != nil { - return nil, fmt.Errorf("failed to get information about Compute Engine region '%s': %v", region, err) - } - - for _, zoneURL := range regionObj.Zones { + for _, zoneURL := range region.Zones { // zone URL example - "https://www.googleapis.com/compute/v1/projects//zones/us-central1-a" zoneNameSs := strings.Split(zoneURL, "/") zoneName := zoneNameSs[len(zoneNameSs)-1] @@ -372,16 +358,50 @@ func (g *GCP) ComputeZonesInRegion(ctx context.Context, region string) ([]string Project: g.GetProjectID(), Zone: zoneName, } - zoneObj, err := zonesClient.Get(ctx, getZoneReq) + zone, err := zonesClient.Get(ctx, getZoneReq) if err != nil { return nil, fmt.Errorf("failed to get information about Compute Engine zone '%s': %v", zoneName, err) } // Make sure to return only Zones, which can be used - if zoneObj.GetStatus() == computepb.Zone_UP.String() { - zones = append(zones, zoneName) + if zone.GetStatus() == computepb.Zone_UP.String() { + zones = append(zones, zone) } } return zones, nil } + +// ComputeRegionsList returns list of GCE regions based on the provided filter. +// +// Uses: +// - Compute Engine API +func (g *GCP) ComputeRegionsList(ctx context.Context, filter string) ([]*computepb.Region, error) { + var regions []*computepb.Region + + regionsClient, err := compute.NewRegionsRESTClient(ctx, option.WithCredentials(g.creds)) + if err != nil { + return nil, fmt.Errorf("failed to get GCE regions client: %v", err) + } + defer regionsClient.Close() + + listRegionsReq := &computepb.ListRegionsRequest{ + Project: g.GetProjectID(), + Filter: common.StringToPtr(filter), + } + regionsIter := regionsClient.List(ctx, listRegionsReq) + + for { + resp, err := regionsIter.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("error while iterating over GCE regions: %v", err) + } + + regions = append(regions, resp) + } + + return regions, nil +}