particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cloud-cleaner: clean up image and vm after GCP integration test

Browse source 
Extend internal GCP library to allow deleting Compute Node image and
instance. In addition provide function to load service account
credentials file content from the environment.

Change names used for GCP image and instance in `api.sh` integration
test to make them predictable. This is important, so that cloud-cleaner
can identify potentially left over resources and clean them up. Use the
same approach for generating predictable, but run-specific, test ID as
in GenerateCIArtifactName() from internal/test/helpers.go. Use SHA224
to generate a hash from the string, because it can contain characters
not allowed by GCP for resource name (specifically "_" e.g. in "x86_64").
SHA-224 was picked because it generates short enough output and it is
future proof for use in RHEL (unlike MD5 or SHA-1).

Refactor cloud-cleaner to clean up GCP resources and also to run cleanup
for each cloud in a separate goroutine.

Modify run_cloud_cleaner.sh to be able to run in environment in which
AZURE_CREDS is not defined.

Always run cloud-cleaner after integration tests for rhel8, rhel84 and
cs8, which test GCP.

Define DISTRO_CODE for each integration testing stage in Jenkinsfile.

Signed-off-by: Tomas Hozza <thozza@redhat.com>
This commit is contained in:
Tomas Hozza 2021-03-10 13:41:54 +01:00 • committed by Tom Gundersen
parent f9fe699564
commit aa1d038b59
6 changed files with 220 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

128
cmd/cloud-cleaner/main.go
View file

@ -3,52 +3,136 @@
package main
import (
"crypto/sha256"
"fmt"
"log"
"os"
"sync"
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/osbuild/osbuild-composer/internal/boot/azuretest"
"github.com/osbuild/osbuild-composer/internal/cloud/gcp"
"github.com/osbuild/osbuild-composer/internal/test"
)
func panicErr(err error) {
func cleanupGCP(testID string, wg *sync.WaitGroup) {
defer wg.Done()
log.Println("[GCP] Running clean up")
GCPRegion, ok := os.LookupEnv("GCP_REGION")
if !ok {
log.Println("[GCP] Error: 'GCP_REGION' is not set in the environment.")
return
}
// api.sh test uses '--zone="$GCP_REGION-a"'
GCPZone := fmt.Sprintf("%s-a", GCPRegion)
// max 62 characters
// Must be a match of regex '[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?|[1-9][0-9]{0,19}'
// use sha224sum to get predictable testID without invalid characters
testIDhash := fmt.Sprintf("%x", sha256.Sum224([]byte(testID)))
// Resource names to clean up
GCPInstance := fmt.Sprintf("vm-%s", testIDhash)
GCPImage := fmt.Sprintf("image-%s", testIDhash)
// It does not matter if there was any error. If the credentials file was
// read successfully then 'creds' should be non-nil, otherwise it will be
// nil. Both values are acceptable for creating a new "GCP" instance.
// If 'creds' is nil, then GCP library will try to authenticate using
// the instance permissions.
creds, err := gcp.GetCredentialsFromEnv()
if err != nil {
panic(err)
log.Printf("[GCP] Error: %v. This may not be an issue.", err)
}
// If this fails, there is no point in continuing
g, err := gcp.New(creds)
if err != nil {
log.Printf("[GCP] Error: %v", err)
return
}
// Try to delete potentially running instance
log.Printf("[GCP] 🧹 Deleting VM instance %s in %s. "+
"This should fail if the test succedded.", GCPInstance, GCPZone)
err = g.ComputeInstanceDelete(GCPZone, GCPInstance)
if err != nil {
log.Printf("[GCP] Error: %v", err)
}
// Try to clean up storage of cache objects after image import job
log.Println("[GCP] 🧹 Cleaning up cache objects from storage after image " +
"import. This should fail if the test succedded.")
cacheObjects, errs := g.StorageImageImportCleanup(GCPImage)
for _, err = range errs {
log.Printf("[GCP] Error: %v", err)
}
for _, cacheObject := range cacheObjects {
log.Printf("[GCP] 🧹 Deleted image import job file %s", cacheObject)
}
// Try to delete the imported image
log.Printf("[GCP] 🧹 Deleting image %s. This should fail if the test succedded.", GCPImage)
err = g.ComputeImageDelete(GCPImage)
if err != nil {
log.Printf("[GCP] Error: %v", err)
}
}
func printErr(err error) {
if err != nil {
fmt.Println(err)
}
}
func cleanupAzure(testID string, wg *sync.WaitGroup) {
defer wg.Done()
func main() {
fmt.Println("Running a cloud cleanup")
log.Println("[Azure] Running clean up")
// Load Azure credentials
creds, err := azuretest.GetAzureCredentialsFromEnv()
panicErr(err)
if creds == nil {
panic("empty credentials")
if err != nil {
log.Printf("[Azure] Error: %v", err)
return
}
// Get test ID
testID, err := test.GenerateCIArtifactName("")
panicErr(err)
if creds == nil {
log.Println("[Azure] Error: empty credentials")
return
}
// Delete the vhd image
imageName := "image-" + testID + ".vhd"
fmt.Println("Running delete image from Azure, this should fail if the test succedded")
log.Println("[Azure] Deleting image. This should fail if the test succedded.")
err = azuretest.DeleteImageFromAzure(creds, imageName)
printErr(err)
if err != nil {
log.Printf("[Azure] Error: %v", err)
}
// Delete all remaining resources (see the full list in the CleanUpBootedVM function)
fmt.Println("Running clean up booted VM, this should fail if the test succedded")
log.Println("[Azure] Cleaning up booted VM. This should fail if the test succedded.")
parameters := azuretest.NewDeploymentParameters(creds, imageName, testID, "")
clientCredentialsConfig := auth.NewClientCredentialsConfig(creds.ClientID, creds.ClientSecret, creds.TenantID)
authorizer, err := clientCredentialsConfig.Authorizer()
panicErr(err)
if err != nil {
log.Printf("[Azure] Error: %v", err)
return
}
err = azuretest.CleanUpBootedVM(creds, parameters, authorizer, testID)
printErr(err)
if err != nil {
log.Printf("[Azure] Error: %v", err)
}
}
func main() {
log.Println("Running a cloud cleanup")
// Get test ID
testID, err := test.GenerateCIArtifactName("")
if err != nil {
log.Fatalf("Failed to get testID: %v", err)
}
var wg sync.WaitGroup
wg.Add(2)
go cleanupAzure(testID, &wg)
go cleanupGCP(testID, &wg)
wg.Wait()
}

37
internal/cloud/gcp/compute.go
View file

@ -204,3 +204,40 @@ func (g *GCP) ComputeImageShare(imageName string, shareWith []string) error {
return nil
}
// ComputeImageDelete deletes a Compute Node image with the given name. If the
// image existed and was successfully deleted, no error is returned.
//
// Uses:
// - Compute Engine API
func (g *GCP) ComputeImageDelete(image string) error {
ctx := context.Background()
computeService, err := compute.NewService(ctx, option.WithCredentials(g.creds))
if err != nil {
return fmt.Errorf("failed to get Compute Engine client: %v", err)
}
_, err = computeService.Images.Delete(g.creds.ProjectID, image).Context(ctx).Do()
return err
}
// ComputeInstanceDelete deletes a Compute Node instance with the given name and
// running in the given zone. If the instance existed and was successfully deleted,
// no error is returned.
//
// Uses:
// - Compute Engine API
func (g *GCP) ComputeInstanceDelete(zone, instance string) error {
ctx := context.Background()
computeService, err := compute.NewService(ctx, option.WithCredentials(g.creds))
if err != nil {
return fmt.Errorf("failed to get Compute Engine client: %v", err)
}
_, err = computeService.Instances.Delete(g.creds.ProjectID, zone, instance).Context(ctx).Do()
return err
}

31
internal/cloud/gcp/gcp.go
View file

@ -3,6 +3,8 @@ package gcp
import (
"context"
"fmt"
"io/ioutil"
"os"
cloudbuild "cloud.google.com/go/cloudbuild/apiv1"
"cloud.google.com/go/storage"
@ -10,6 +12,12 @@ import (
"google.golang.org/api/compute/v1"
)
// GCPCredentialsEnvName contains name of the environment variable used
// to specify the path to file with CGP service account credentials
const (
GCPCredentialsEnvName string = "GOOGLE_APPLICATION_CREDENTIALS"
)
// GCP structure holds necessary information to authenticate and interact with GCP.
type GCP struct {
creds *google.Credentials
@ -49,6 +57,29 @@ func New(credentials []byte) (*GCP, error) {
return &GCP{creds}, nil
}
// GetCredentialsFromEnv reads the service account credentials JSON file from
// the path pointed to by the environment variable name stored in
// 'GCPCredentialsEnvName'. If the content of the JSON file was read successfully,
// its content is returned as []byte, otherwise nil is returned with proper error.
func GetCredentialsFromEnv() ([]byte, error) {
credsPath, exists := os.LookupEnv(GCPCredentialsEnvName)
if !exists {
return nil, fmt.Errorf("'%s' env variable is not set", GCPCredentialsEnvName)
}
if credsPath == "" {
return nil, fmt.Errorf("'%s' env variable is empty", GCPCredentialsEnvName)
}
var err error
credentials, err := ioutil.ReadFile(credsPath)
if err != nil {
return nil, fmt.Errorf("Error while reading credentials file: %s", err)
}
return credentials, nil
}
// GetProjectID returns a string with the Project ID of the project, used for
// all GCP operations.
func (g *GCP) GetProjectID() string {

29
schutzbot/Jenkinsfile vendored
View file

@ -272,12 +272,12 @@ pipeline {
agent { label "f32cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
DISTRO_CODE = "fedora32"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AZURE_CREDS = credentials('azure')
OPENSTACK_CREDS = credentials("psi-openstack-creds")
VCENTER_CREDS = credentials('vmware-vcenter-credentials')
DISTRO_CODE = "fedora32"
}
steps {
run_tests('image')
@ -302,6 +302,7 @@ pipeline {
agent { label "f32cloudbase && x86_64 && aws" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "fedora32"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
@ -379,12 +380,12 @@ pipeline {
agent { label "f33cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
DISTRO_CODE = "fedora33"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AZURE_CREDS = credentials('azure')
OPENSTACK_CREDS = credentials("psi-openstack-creds")
VCENTER_CREDS = credentials('vmware-vcenter-credentials')
DISTRO_CODE = "fedora33"
}
steps {
run_tests('image')
@ -409,6 +410,7 @@ pipeline {
agent { label "f33cloudbase && x86_64 && aws" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "fedora33"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
@ -486,11 +488,11 @@ pipeline {
agent { label "f33cloudbase && aarch64 && aws" }
environment {
TEST_TYPE = "image"
DISTRO_CODE = "fedora33"
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AZURE_CREDS = credentials('azure')
OPENSTACK_CREDS = credentials("psi-openstack-creds")
VCENTER_CREDS = credentials('vmware-vcenter-credentials')
DISTRO_CODE = "fedora33"
}
steps {
run_tests('image')
@ -553,13 +555,13 @@ pipeline {
agent { label "rhel8cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
DISTRO_CODE = "rhel8"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AZURE_CREDS = credentials('azure')
OPENSTACK_CREDS = credentials("psi-openstack-creds")
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
VCENTER_CREDS = credentials('vmware-vcenter-credentials')
DISTRO_CODE = "rhel8"
}
steps {
run_tests('image')
@ -584,6 +586,7 @@ pipeline {
agent { label "rhel8cloudbase && x86_64 && psi" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "rhel8"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
@ -600,6 +603,10 @@ pipeline {
post {
always {
preserve_logs('rhel8-integration')
sh (
label: "Run cloud cleaner just in case something failed",
script: "schutzbot/run_cloud_cleaner.sh"
)
}
}
}
@ -693,12 +700,12 @@ pipeline {
agent { label "rhel84cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
DISTRO_CODE = "rhel84"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AZURE_CREDS = credentials('azure')
OPENSTACK_CREDS = credentials("psi-openstack-creds")
VCENTER_CREDS = credentials('vmware-vcenter-credentials')
DISTRO_CODE = "rhel84"
}
steps {
run_tests('image')
@ -717,6 +724,7 @@ pipeline {
agent { label "rhel84cloudbase && x86_64 && psi" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "rhel84"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
@ -732,6 +740,10 @@ pipeline {
post {
always {
preserve_logs('rhel84-integration')
sh (
label: "Run cloud cleaner just in case something failed",
script: "schutzbot/run_cloud_cleaner.sh"
)
}
}
}
@ -787,12 +799,12 @@ pipeline {
agent { label "cs8cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
DISTRO_CODE = "centos-stream8"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AZURE_CREDS = credentials('azure')
OPENSTACK_CREDS = credentials("psi-openstack-creds")
VCENTER_CREDS = credentials('vmware-vcenter-credentials')
DISTRO_CODE = "centos-stream8"
}
steps {
run_tests('image')
@ -817,6 +829,7 @@ pipeline {
agent { label "cs8cloudbase && x86_64 && psi" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "centos-stream8"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
@ -831,6 +844,10 @@ pipeline {
post {
always {
preserve_logs('cs8-integration')
sh (
label: "Run cloud cleaner just in case something failed",
script: "schutzbot/run_cloud_cleaner.sh"
)
}
}
}

2
schutzbot/run_cloud_cleaner.sh
View file

@ -1,7 +1,7 @@
#!/bin/bash
set -euo pipefail
CLEANER_CMD="env $(cat "$AZURE_CREDS") BRANCH_NAME=$BRANCH_NAME BUILD_ID=$BUILD_ID DISTRO_CODE=$DISTRO_CODE /usr/libexec/osbuild-composer-test/cloud-cleaner"
CLEANER_CMD="env $(cat "${AZURE_CREDS:-/dev/null}") BRANCH_NAME=$BRANCH_NAME BUILD_ID=$BUILD_ID DISTRO_CODE=$DISTRO_CODE /usr/libexec/osbuild-composer-test/cloud-cleaner"
echo "🧹 Running the cloud cleaner"
$CLEANER_CMD

24
test/cases/api.sh
View file

@ -252,6 +252,18 @@ REQUEST_FILE="${WORKDIR}/request.json"
ARCH=$(uname -m)
SSH_USER=
# Generate a string, which can be used as a predictable resource name,
# especially when running the test in Jenkins where we may need to clean up
# resources in case the test unexpectedly fails or is canceled
JENKINS_HOME="${JENKINS_HOME:-}"
if [[ -n "$JENKINS_HOME" ]]; then
# in Jenkins, imitate GenerateCIArtifactName() from internal/test/helpers.go
TEST_ID="$DISTRO_CODE-$ARCH-$BRANCH_NAME-$BUILD_ID"
else
# if not running in Jenkins, generate ID not relying on specific env variables
TEST_ID=$(uuidgen);
fi
case $(set +x; . /etc/os-release; echo "$ID-$VERSION_ID") in
"rhel-8.4")
DISTRO="rhel-84"
@ -315,7 +327,14 @@ EOF
}
function createReqFileGCP() {
GCP_IMAGE_NAME="image-$(uuidgen)"
# constrains for GCP resource IDs:
# - max 62 characters
# - must be a match of regex '[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?|[1-9][0-9]{0,19}'
#
# use sha224sum to get predictable 56 characters long testID without invalid characters
GCP_TEST_ID_HASH="$(echo -n "$TEST_ID" | sha224sum - | sed -E 's/([a-z0-9])\s+-/\1/')"
GCP_IMAGE_NAME="image-$GCP_TEST_ID_HASH"
cat > "$REQUEST_FILE" << EOF
{
@ -577,7 +596,8 @@ function verifyInGCP() {
echo "${SSH_USER}:$(cat "$GCP_SSH_KEY".pub)" > "$GCP_SSH_METADATA_FILE"
# create the instance
GCP_INSTANCE_NAME="gcp-instance-$(uuidgen)"
# resource ID can have max 62 characters, the $GCP_TEST_ID_HASH contains 56 characters
GCP_INSTANCE_NAME="vm-$GCP_TEST_ID_HASH"
$GCP_CMD compute instances create "$GCP_INSTANCE_NAME" \
--zone="$GCP_REGION-a" \