diff --git a/cmd/cloud-cleaner/main.go b/cmd/cloud-cleaner/main.go index 93daca16d..4f702f869 100644 --- a/cmd/cloud-cleaner/main.go +++ b/cmd/cloud-cleaner/main.go @@ -3,52 +3,136 @@ package main import ( + "crypto/sha256" "fmt" + "log" + "os" + "sync" "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/osbuild/osbuild-composer/internal/boot/azuretest" + "github.com/osbuild/osbuild-composer/internal/cloud/gcp" "github.com/osbuild/osbuild-composer/internal/test" ) -func panicErr(err error) { +func cleanupGCP(testID string, wg *sync.WaitGroup) { + defer wg.Done() + + log.Println("[GCP] Running clean up") + + GCPRegion, ok := os.LookupEnv("GCP_REGION") + if !ok { + log.Println("[GCP] Error: 'GCP_REGION' is not set in the environment.") + return + } + // api.sh test uses '--zone="$GCP_REGION-a"' + GCPZone := fmt.Sprintf("%s-a", GCPRegion) + // max 62 characters + // Must be a match of regex '[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?|[1-9][0-9]{0,19}' + // use sha224sum to get predictable testID without invalid characters + testIDhash := fmt.Sprintf("%x", sha256.Sum224([]byte(testID))) + + // Resource names to clean up + GCPInstance := fmt.Sprintf("vm-%s", testIDhash) + GCPImage := fmt.Sprintf("image-%s", testIDhash) + + // It does not matter if there was any error. If the credentials file was + // read successfully then 'creds' should be non-nil, otherwise it will be + // nil. Both values are acceptable for creating a new "GCP" instance. + // If 'creds' is nil, then GCP library will try to authenticate using + // the instance permissions. + creds, err := gcp.GetCredentialsFromEnv() if err != nil { - panic(err) + log.Printf("[GCP] Error: %v. This may not be an issue.", err) + } + + // If this fails, there is no point in continuing + g, err := gcp.New(creds) + if err != nil { + log.Printf("[GCP] Error: %v", err) + return + } + + // Try to delete potentially running instance + log.Printf("[GCP] 🧹 Deleting VM instance %s in %s. "+ + "This should fail if the test succedded.", GCPInstance, GCPZone) + err = g.ComputeInstanceDelete(GCPZone, GCPInstance) + if err != nil { + log.Printf("[GCP] Error: %v", err) + } + + // Try to clean up storage of cache objects after image import job + log.Println("[GCP] 🧹 Cleaning up cache objects from storage after image " + + "import. This should fail if the test succedded.") + cacheObjects, errs := g.StorageImageImportCleanup(GCPImage) + for _, err = range errs { + log.Printf("[GCP] Error: %v", err) + } + for _, cacheObject := range cacheObjects { + log.Printf("[GCP] 🧹 Deleted image import job file %s", cacheObject) + } + + // Try to delete the imported image + log.Printf("[GCP] 🧹 Deleting image %s. This should fail if the test succedded.", GCPImage) + err = g.ComputeImageDelete(GCPImage) + if err != nil { + log.Printf("[GCP] Error: %v", err) } } -func printErr(err error) { - if err != nil { - fmt.Println(err) - } -} +func cleanupAzure(testID string, wg *sync.WaitGroup) { + defer wg.Done() - - -func main() { - fmt.Println("Running a cloud cleanup") + log.Println("[Azure] Running clean up") // Load Azure credentials creds, err := azuretest.GetAzureCredentialsFromEnv() - panicErr(err) - if creds == nil { - panic("empty credentials") + if err != nil { + log.Printf("[Azure] Error: %v", err) + return } - // Get test ID - testID, err := test.GenerateCIArtifactName("") - panicErr(err) + if creds == nil { + log.Println("[Azure] Error: empty credentials") + return + } + // Delete the vhd image imageName := "image-" + testID + ".vhd" - fmt.Println("Running delete image from Azure, this should fail if the test succedded") + log.Println("[Azure] Deleting image. This should fail if the test succedded.") err = azuretest.DeleteImageFromAzure(creds, imageName) - printErr(err) + if err != nil { + log.Printf("[Azure] Error: %v", err) + } // Delete all remaining resources (see the full list in the CleanUpBootedVM function) - fmt.Println("Running clean up booted VM, this should fail if the test succedded") + log.Println("[Azure] Cleaning up booted VM. This should fail if the test succedded.") parameters := azuretest.NewDeploymentParameters(creds, imageName, testID, "") clientCredentialsConfig := auth.NewClientCredentialsConfig(creds.ClientID, creds.ClientSecret, creds.TenantID) authorizer, err := clientCredentialsConfig.Authorizer() - panicErr(err) + if err != nil { + log.Printf("[Azure] Error: %v", err) + return + } + err = azuretest.CleanUpBootedVM(creds, parameters, authorizer, testID) - printErr(err) + if err != nil { + log.Printf("[Azure] Error: %v", err) + } +} + +func main() { + log.Println("Running a cloud cleanup") + + // Get test ID + testID, err := test.GenerateCIArtifactName("") + if err != nil { + log.Fatalf("Failed to get testID: %v", err) + } + + var wg sync.WaitGroup + wg.Add(2) + go cleanupAzure(testID, &wg) + go cleanupGCP(testID, &wg) + wg.Wait() } diff --git a/internal/cloud/gcp/compute.go b/internal/cloud/gcp/compute.go index ba8e52a20..80b5638d6 100644 --- a/internal/cloud/gcp/compute.go +++ b/internal/cloud/gcp/compute.go @@ -204,3 +204,40 @@ func (g *GCP) ComputeImageShare(imageName string, shareWith []string) error { return nil } + +// ComputeImageDelete deletes a Compute Node image with the given name. If the +// image existed and was successfully deleted, no error is returned. +// +// Uses: +// - Compute Engine API +func (g *GCP) ComputeImageDelete(image string) error { + ctx := context.Background() + + computeService, err := compute.NewService(ctx, option.WithCredentials(g.creds)) + if err != nil { + return fmt.Errorf("failed to get Compute Engine client: %v", err) + } + + _, err = computeService.Images.Delete(g.creds.ProjectID, image).Context(ctx).Do() + + return err +} + +// ComputeInstanceDelete deletes a Compute Node instance with the given name and +// running in the given zone. If the instance existed and was successfully deleted, +// no error is returned. +// +// Uses: +// - Compute Engine API +func (g *GCP) ComputeInstanceDelete(zone, instance string) error { + ctx := context.Background() + + computeService, err := compute.NewService(ctx, option.WithCredentials(g.creds)) + if err != nil { + return fmt.Errorf("failed to get Compute Engine client: %v", err) + } + + _, err = computeService.Instances.Delete(g.creds.ProjectID, zone, instance).Context(ctx).Do() + + return err +} diff --git a/internal/cloud/gcp/gcp.go b/internal/cloud/gcp/gcp.go index 35b8798b8..58adeaa86 100644 --- a/internal/cloud/gcp/gcp.go +++ b/internal/cloud/gcp/gcp.go @@ -3,6 +3,8 @@ package gcp import ( "context" "fmt" + "io/ioutil" + "os" cloudbuild "cloud.google.com/go/cloudbuild/apiv1" "cloud.google.com/go/storage" @@ -10,6 +12,12 @@ import ( "google.golang.org/api/compute/v1" ) +// GCPCredentialsEnvName contains name of the environment variable used +// to specify the path to file with CGP service account credentials +const ( + GCPCredentialsEnvName string = "GOOGLE_APPLICATION_CREDENTIALS" +) + // GCP structure holds necessary information to authenticate and interact with GCP. type GCP struct { creds *google.Credentials @@ -49,6 +57,29 @@ func New(credentials []byte) (*GCP, error) { return &GCP{creds}, nil } +// GetCredentialsFromEnv reads the service account credentials JSON file from +// the path pointed to by the environment variable name stored in +// 'GCPCredentialsEnvName'. If the content of the JSON file was read successfully, +// its content is returned as []byte, otherwise nil is returned with proper error. +func GetCredentialsFromEnv() ([]byte, error) { + credsPath, exists := os.LookupEnv(GCPCredentialsEnvName) + + if !exists { + return nil, fmt.Errorf("'%s' env variable is not set", GCPCredentialsEnvName) + } + if credsPath == "" { + return nil, fmt.Errorf("'%s' env variable is empty", GCPCredentialsEnvName) + } + + var err error + credentials, err := ioutil.ReadFile(credsPath) + if err != nil { + return nil, fmt.Errorf("Error while reading credentials file: %s", err) + } + + return credentials, nil +} + // GetProjectID returns a string with the Project ID of the project, used for // all GCP operations. func (g *GCP) GetProjectID() string { diff --git a/schutzbot/Jenkinsfile b/schutzbot/Jenkinsfile index 12bfa25d7..3ea33c48d 100644 --- a/schutzbot/Jenkinsfile +++ b/schutzbot/Jenkinsfile @@ -272,12 +272,12 @@ pipeline { agent { label "f32cloudbase && psi && x86_64" } environment { TEST_TYPE = "image" + DISTRO_CODE = "fedora32" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") VCENTER_CREDS = credentials('vmware-vcenter-credentials') - DISTRO_CODE = "fedora32" } steps { run_tests('image') @@ -302,6 +302,7 @@ pipeline { agent { label "f32cloudbase && x86_64 && aws" } environment { TEST_TYPE = "integration" + DISTRO_CODE = "fedora32" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') @@ -379,12 +380,12 @@ pipeline { agent { label "f33cloudbase && psi && x86_64" } environment { TEST_TYPE = "image" + DISTRO_CODE = "fedora33" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") VCENTER_CREDS = credentials('vmware-vcenter-credentials') - DISTRO_CODE = "fedora33" } steps { run_tests('image') @@ -409,6 +410,7 @@ pipeline { agent { label "f33cloudbase && x86_64 && aws" } environment { TEST_TYPE = "integration" + DISTRO_CODE = "fedora33" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') @@ -486,11 +488,11 @@ pipeline { agent { label "f33cloudbase && aarch64 && aws" } environment { TEST_TYPE = "image" + DISTRO_CODE = "fedora33" AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") VCENTER_CREDS = credentials('vmware-vcenter-credentials') - DISTRO_CODE = "fedora33" } steps { run_tests('image') @@ -553,13 +555,13 @@ pipeline { agent { label "rhel8cloudbase && psi && x86_64" } environment { TEST_TYPE = "image" + DISTRO_CODE = "rhel8" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production') VCENTER_CREDS = credentials('vmware-vcenter-credentials') - DISTRO_CODE = "rhel8" } steps { run_tests('image') @@ -584,6 +586,7 @@ pipeline { agent { label "rhel8cloudbase && x86_64 && psi" } environment { TEST_TYPE = "integration" + DISTRO_CODE = "rhel8" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production') @@ -600,6 +603,10 @@ pipeline { post { always { preserve_logs('rhel8-integration') + sh ( + label: "Run cloud cleaner just in case something failed", + script: "schutzbot/run_cloud_cleaner.sh" + ) } } } @@ -693,12 +700,12 @@ pipeline { agent { label "rhel84cloudbase && psi && x86_64" } environment { TEST_TYPE = "image" + DISTRO_CODE = "rhel84" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") VCENTER_CREDS = credentials('vmware-vcenter-credentials') - DISTRO_CODE = "rhel84" } steps { run_tests('image') @@ -717,6 +724,7 @@ pipeline { agent { label "rhel84cloudbase && x86_64 && psi" } environment { TEST_TYPE = "integration" + DISTRO_CODE = "rhel84" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') @@ -732,6 +740,10 @@ pipeline { post { always { preserve_logs('rhel84-integration') + sh ( + label: "Run cloud cleaner just in case something failed", + script: "schutzbot/run_cloud_cleaner.sh" + ) } } } @@ -787,12 +799,12 @@ pipeline { agent { label "cs8cloudbase && psi && x86_64" } environment { TEST_TYPE = "image" + DISTRO_CODE = "centos-stream8" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AZURE_CREDS = credentials('azure') OPENSTACK_CREDS = credentials("psi-openstack-creds") VCENTER_CREDS = credentials('vmware-vcenter-credentials') - DISTRO_CODE = "centos-stream8" } steps { run_tests('image') @@ -817,6 +829,7 @@ pipeline { agent { label "cs8cloudbase && x86_64 && psi" } environment { TEST_TYPE = "integration" + DISTRO_CODE = "centos-stream8" AWS_CREDS = credentials('aws-credentials-osbuildci') AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test') AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account') @@ -831,6 +844,10 @@ pipeline { post { always { preserve_logs('cs8-integration') + sh ( + label: "Run cloud cleaner just in case something failed", + script: "schutzbot/run_cloud_cleaner.sh" + ) } } } diff --git a/schutzbot/run_cloud_cleaner.sh b/schutzbot/run_cloud_cleaner.sh index 4715a91bd..546cff8d5 100755 --- a/schutzbot/run_cloud_cleaner.sh +++ b/schutzbot/run_cloud_cleaner.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -CLEANER_CMD="env $(cat "$AZURE_CREDS") BRANCH_NAME=$BRANCH_NAME BUILD_ID=$BUILD_ID DISTRO_CODE=$DISTRO_CODE /usr/libexec/osbuild-composer-test/cloud-cleaner" +CLEANER_CMD="env $(cat "${AZURE_CREDS:-/dev/null}") BRANCH_NAME=$BRANCH_NAME BUILD_ID=$BUILD_ID DISTRO_CODE=$DISTRO_CODE /usr/libexec/osbuild-composer-test/cloud-cleaner" echo "🧹 Running the cloud cleaner" $CLEANER_CMD diff --git a/test/cases/api.sh b/test/cases/api.sh index 98cdbd34c..ee110043b 100755 --- a/test/cases/api.sh +++ b/test/cases/api.sh @@ -252,6 +252,18 @@ REQUEST_FILE="${WORKDIR}/request.json" ARCH=$(uname -m) SSH_USER= +# Generate a string, which can be used as a predictable resource name, +# especially when running the test in Jenkins where we may need to clean up +# resources in case the test unexpectedly fails or is canceled +JENKINS_HOME="${JENKINS_HOME:-}" +if [[ -n "$JENKINS_HOME" ]]; then + # in Jenkins, imitate GenerateCIArtifactName() from internal/test/helpers.go + TEST_ID="$DISTRO_CODE-$ARCH-$BRANCH_NAME-$BUILD_ID" +else + # if not running in Jenkins, generate ID not relying on specific env variables + TEST_ID=$(uuidgen); +fi + case $(set +x; . /etc/os-release; echo "$ID-$VERSION_ID") in "rhel-8.4") DISTRO="rhel-84" @@ -315,7 +327,14 @@ EOF } function createReqFileGCP() { - GCP_IMAGE_NAME="image-$(uuidgen)" + # constrains for GCP resource IDs: + # - max 62 characters + # - must be a match of regex '[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?|[1-9][0-9]{0,19}' + # + # use sha224sum to get predictable 56 characters long testID without invalid characters + GCP_TEST_ID_HASH="$(echo -n "$TEST_ID" | sha224sum - | sed -E 's/([a-z0-9])\s+-/\1/')" + + GCP_IMAGE_NAME="image-$GCP_TEST_ID_HASH" cat > "$REQUEST_FILE" << EOF { @@ -577,7 +596,8 @@ function verifyInGCP() { echo "${SSH_USER}:$(cat "$GCP_SSH_KEY".pub)" > "$GCP_SSH_METADATA_FILE" # create the instance - GCP_INSTANCE_NAME="gcp-instance-$(uuidgen)" + # resource ID can have max 62 characters, the $GCP_TEST_ID_HASH contains 56 characters + GCP_INSTANCE_NAME="vm-$GCP_TEST_ID_HASH" $GCP_CMD compute instances create "$GCP_INSTANCE_NAME" \ --zone="$GCP_REGION-a" \