diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b63891e1c..234ee49d2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -442,7 +442,7 @@ API: RUNNER: - aws/fedora-35-x86_64 -libvirt: +.libvirt_integration: stage: test extends: .terraform/openstack rules: @@ -450,7 +450,7 @@ libvirt: - !reference [.nightly_rules, rules] script: - schutzbot/deploy.sh - - /usr/libexec/tests/osbuild-composer/libvirt.sh + - /usr/libexec/tests/osbuild-composer/${SCRIPT} parallel: matrix: - RUNNER: @@ -460,6 +460,21 @@ libvirt: - rhos-01/rhel-9.0-nightly-x86_64 - rhos-01/centos-stream-9-x86_64 +libvirt.sh: + extends: .libvirt_integration + variables: + SCRIPT: libvirt.sh + +generic_s3.sh: + extends: .libvirt_integration + variables: + SCRIPT: generic_s3.sh + +aws_s3.sh: + extends: .libvirt_integration + variables: + SCRIPT: aws_s3.sh + RHEL 9 on 8: stage: test extends: .terraform diff --git a/Makefile b/Makefile index 3df7af7c6..b554c32d5 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ man: $(MANPAGES_TROFF) .PHONY: build build: - - mkdir bin + - mkdir -p bin go build -o bin/osbuild-composer ./cmd/osbuild-composer/ go build -o bin/osbuild-worker ./cmd/osbuild-worker/ go build -o bin/osbuild-pipeline ./cmd/osbuild-pipeline/ @@ -116,6 +116,7 @@ build: go build -o bin/osbuild-upload-aws ./cmd/osbuild-upload-aws/ go build -o bin/osbuild-upload-gcp ./cmd/osbuild-upload-gcp/ go build -o bin/osbuild-upload-oci ./cmd/osbuild-upload-oci/ + go build -o bin/osbuild-upload-generic-s3 ./cmd/osbuild-upload-generic-s3/ go build -o bin/osbuild-mock-openid-provider ./cmd/osbuild-mock-openid-provider go build -o bin/osbuild-service-maintenance ./cmd/osbuild-service-maintenance go test -c -tags=integration -o bin/osbuild-composer-cli-tests ./cmd/osbuild-composer-cli-tests/main_test.go diff --git a/cmd/osbuild-upload-generic-s3/main.go b/cmd/osbuild-upload-generic-s3/main.go new file mode 100644 index 000000000..956daebc1 --- /dev/null +++ b/cmd/osbuild-upload-generic-s3/main.go @@ -0,0 +1,44 @@ +package main + +import ( + "flag" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + + "github.com/osbuild/osbuild-composer/internal/cloud/awscloud" +) + +func main() { + var accessKeyID string + var secretAccessKey string + var sessionToken string + var region string + var endpoint string + var bucketName string + var keyName string + var filename string + flag.StringVar(&accessKeyID, "access-key-id", "", "access key ID") + flag.StringVar(&secretAccessKey, "secret-access-key", "", "secret access key") + flag.StringVar(&sessionToken, "session-token", "", "session token") + flag.StringVar(®ion, "region", "", "target region") + flag.StringVar(&endpoint, "endpoint", "", "target endpoint") + flag.StringVar(&bucketName, "bucket", "", "target S3 bucket name") + flag.StringVar(&keyName, "key", "", "target S3 key name") + flag.StringVar(&filename, "image", "", "image file to upload") + flag.Parse() + + a, err := awscloud.NewForEndpoint(endpoint, region, accessKeyID, secretAccessKey, sessionToken) + if err != nil { + println(err.Error()) + return + } + + uploadOutput, err := a.Upload(filename, bucketName, keyName) + if err != nil { + println(err.Error()) + return + } + + fmt.Printf("file uploaded to %s\n", aws.StringValue(&uploadOutput.Location)) +} diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index 18bc0fff5..afab89163 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -30,13 +30,14 @@ import ( ) type OSBuildJobImpl struct { - Store string - Output string - KojiServers map[string]koji.GSSAPICredentials - GCPCreds []byte - AzureCreds *azure.Credentials - AWSCreds string - AWSBucket string + Store string + Output string + KojiServers map[string]koji.GSSAPICredentials + GCPCreds []byte + AzureCreds *azure.Credentials + AWSCreds string + AWSBucket string + GenericS3Creds string } // Returns an *awscloud.AWS object with the credentials of the request. If they @@ -52,6 +53,16 @@ func (impl *OSBuildJobImpl) getAWS(region string, accessId string, secret string } } +func (impl *OSBuildJobImpl) getAWSForEndpoint(endpoint, region, accessId, secret, token string) (*awscloud.AWS, error) { + if accessId != "" && secret != "" { + return awscloud.NewForEndpoint(endpoint, region, accessId, secret, token) + } + if impl.GenericS3Creds != "" { + return awscloud.NewForEndpointFromFile(impl.GenericS3Creds, endpoint, region) + } + return nil, fmt.Errorf("no credentials found") +} + func validateResult(result *worker.OSBuildJobResult, jobID string) { logWithId := logrus.WithField("jobId", jobID) if result.JobError != nil { @@ -71,6 +82,65 @@ func validateResult(result *worker.OSBuildJobResult, jobID string) { result.Success = true } +func uploadToS3(a *awscloud.AWS, outputDirectory, exportPath, bucket, key, filename string, osbuildJobResult *worker.OSBuildJobResult, genericS3 bool, streamOptimized bool, streamOptimizedPath string) (err error) { + imagePath := path.Join(outputDirectory, exportPath, filename) + + // *** SPECIAL VMDK HANDLING START *** + // Upload the VMDK image as stream-optimized. + // The VMDK conversion is applied only when the job was submitted by Weldr API, + // therefore we need to do the conversion here explicitly if it was not done. + if streamOptimized { + // If the streamOptimizedPath is empty, the conversion was not done + if streamOptimizedPath == "" { + var f *os.File + f, err = vmware.OpenAsStreamOptimizedVmdk(imagePath) + if err != nil { + osbuildJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidConfig, err.Error()) + return nil + } + streamOptimizedPath = f.Name() + f.Close() + } + // Replace the original file by the stream-optimized one + err = os.Rename(streamOptimizedPath, imagePath) + if err != nil { + osbuildJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidConfig, err.Error()) + return nil + } + } + // *** SPECIAL VMDK HANDLING END *** + + if key == "" { + key = uuid.New().String() + } + key += "-" + filename + + _, err = a.Upload(imagePath, bucket, key) + if err != nil { + osbuildJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, err.Error()) + return + } + + url, err := a.S3ObjectPresignedURL(bucket, key) + if err != nil { + osbuildJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, err.Error()) + return + } + + var targetResult *target.TargetResult + if genericS3 { + targetResult = target.NewGenericS3TargetResult(&target.GenericS3TargetResultOptions{URL: url}) + } else { + targetResult = target.NewAWSS3TargetResult(&target.AWSS3TargetResultOptions{URL: url}) + } + osbuildJobResult.TargetResults = append(osbuildJobResult.TargetResults, targetResult) + + osbuildJobResult.Success = true + osbuildJobResult.UploadStatus = "success" + + return +} + func (impl *OSBuildJobImpl) Run(job worker.Job) error { logWithId := logrus.WithField("jobId", job.Id().String()) // Initialize variable needed for reporting back to osbuild-composer. @@ -314,59 +384,26 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { return nil } - key := options.Key - if key == "" { - key = uuid.New().String() - } - key += "-" + options.Filename - bucket := options.Bucket if impl.AWSBucket != "" { bucket = impl.AWSBucket } - imagePath := path.Join(outputDirectory, exportPath, options.Filename) - - // *** SPECIAL VMDK HANDLING START *** - // Upload the VMDK image as stream-optimized. - // The VMDK conversion is applied only when the job was submitted by Weldr API, - // therefore we need to do the conversion here explicitly if it was not done. - if args.StreamOptimized { - // If the streamOptimizedPath is empty, the conversion was not done - if streamOptimizedPath == "" { - var f *os.File - f, err = vmware.OpenAsStreamOptimizedVmdk(imagePath) - if err != nil { - osbuildJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidConfig, err.Error()) - return nil - } - streamOptimizedPath = f.Name() - f.Close() - } - // Replace the original file by the stream-optimized one - err = os.Rename(streamOptimizedPath, imagePath) - if err != nil { - osbuildJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidConfig, err.Error()) - return nil - } - } - // *** SPECIAL VMDK HANDLING END *** - - _, err = a.Upload(imagePath, bucket, key) + err = uploadToS3(a, outputDirectory, exportPath, bucket, options.Key, options.Filename, osbuildJobResult, false, args.StreamOptimized, streamOptimizedPath) if err != nil { - osbuildJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, err.Error()) return nil } - url, err := a.S3ObjectPresignedURL(bucket, key) + case *target.GenericS3TargetOptions: + a, err := impl.getAWSForEndpoint(options.Endpoint, options.Region, options.AccessKeyID, options.SecretAccessKey, options.SessionToken) if err != nil { - osbuildJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, err.Error()) + osbuildJobResult.JobError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidConfig, err.Error()) return nil } - osbuildJobResult.TargetResults = append(osbuildJobResult.TargetResults, target.NewAWSS3TargetResult(&target.AWSS3TargetResultOptions{URL: url})) - - osbuildJobResult.Success = true - osbuildJobResult.UploadStatus = "success" + err = uploadToS3(a, outputDirectory, exportPath, options.Bucket, options.Key, options.Filename, osbuildJobResult, true, args.StreamOptimized, streamOptimizedPath) + if err != nil { + return nil + } case *target.AzureTargetOptions: azureStorageClient, err := azure.NewStorageClient(options.StorageAccount, options.StorageAccessKey) if err != nil { diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go index e6cacb7d6..96b5fae60 100644 --- a/cmd/osbuild-worker/main.go +++ b/cmd/osbuild-worker/main.go @@ -139,6 +139,9 @@ func main() { Credentials string `toml:"credentials"` Bucket string `toml:"bucket"` } `toml:"aws"` + GenericS3 *struct { + Credentials string `toml:"credentials"` + } `toml:"generic_s3"` Authentication *struct { OAuthURL string `toml:"oauth_url"` OfflineTokenPath string `toml:"offline_token"` @@ -306,6 +309,11 @@ func main() { awsBucket = config.AWS.Bucket } + var genericS3Credentials = "" + if config.GenericS3 != nil { + genericS3Credentials = config.GenericS3.Credentials + } + // depsolve jobs can be done during other jobs depsolveCtx, depsolveCtxCancel := context.WithCancel(context.Background()) defer depsolveCtxCancel() @@ -340,13 +348,14 @@ func main() { // non-depsolve job jobImpls := map[string]JobImplementation{ "osbuild": &OSBuildJobImpl{ - Store: store, - Output: output, - KojiServers: kojiServers, - GCPCreds: gcpCredentials, - AzureCreds: azureCredentials, - AWSCreds: awsCredentials, - AWSBucket: awsBucket, + Store: store, + Output: output, + KojiServers: kojiServers, + GCPCreds: gcpCredentials, + AzureCreds: azureCredentials, + AWSCreds: awsCredentials, + AWSBucket: awsBucket, + GenericS3Creds: genericS3Credentials, }, "osbuild-koji": &OSBuildKojiJobImpl{ Store: store, diff --git a/internal/cloud/awscloud/awscloud.go b/internal/cloud/awscloud/awscloud.go index e2282ef08..11c5017ab 100644 --- a/internal/cloud/awscloud/awscloud.go +++ b/internal/cloud/awscloud/awscloud.go @@ -22,11 +22,14 @@ type AWS struct { } // Create a new session from the credentials and the region and returns an *AWS object initialized with it. -func newAwsFromCreds(creds *credentials.Credentials, region string) (*AWS, error) { +func newAwsFromCreds(creds *credentials.Credentials, region string, endpoint *string) (*AWS, error) { // Create a Session with a custom region + s3ForcePathStyle := endpoint != nil sess, err := session.NewSession(&aws.Config{ - Credentials: creds, - Region: aws.String(region), + Credentials: creds, + Region: aws.String(region), + Endpoint: endpoint, + S3ForcePathStyle: &s3ForcePathStyle, }) if err != nil { return nil, err @@ -41,7 +44,7 @@ func newAwsFromCreds(creds *credentials.Credentials, region string) (*AWS, error // Initialize a new AWS object from individual bits. SessionToken is optional func New(region string, accessKeyID string, accessKey string, sessionToken string) (*AWS, error) { - return newAwsFromCreds(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region) + return newAwsFromCreds(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region, nil) } // Initializes a new AWS object with the credentials info found at filename's location. @@ -54,13 +57,31 @@ func New(region string, accessKeyID string, accessKey string, sessionToken strin // "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to // $HOME/.aws/credentials. func NewFromFile(filename string, region string) (*AWS, error) { - return newAwsFromCreds(credentials.NewSharedCredentials(filename, "default"), region) + return newAwsFromCreds(credentials.NewSharedCredentials(filename, "default"), region, nil) } // Initialize a new AWS object from defaults. // Looks for env variables, shared credential file, and EC2 Instance Roles. func NewDefault(region string) (*AWS, error) { - return newAwsFromCreds(nil, region) + return newAwsFromCreds(nil, region, nil) +} + +// Initialize a new AWS object targeting a specific endpoint from individual bits. SessionToken is optional +func NewForEndpoint(endpoint, region string, accessKeyID string, accessKey string, sessionToken string) (*AWS, error) { + return newAwsFromCreds(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region, &endpoint) +} + +// Initializes a new AWS object targeting a specific endpoint with the credentials info found at filename's location. +// The credential files should match the AWS format, such as: +// [default] +// aws_access_key_id = secretString1 +// aws_secret_access_key = secretString2 +// +// If filename is empty the underlying function will look for the +// "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to +// $HOME/.aws/credentials. +func NewForEndpointFromFile(filename string, endpoint, region string) (*AWS, error) { + return newAwsFromCreds(credentials.NewSharedCredentials(filename, "default"), region, &endpoint) } func (a *AWS) Upload(filename, bucket, key string) (*s3manager.UploadOutput, error) { diff --git a/internal/target/generic_s3_target.go b/internal/target/generic_s3_target.go new file mode 100644 index 000000000..555075751 --- /dev/null +++ b/internal/target/generic_s3_target.go @@ -0,0 +1,20 @@ +package target + +type GenericS3TargetOptions struct { + AWSS3TargetOptions + Endpoint string `json:"endpoint"` +} + +func (GenericS3TargetOptions) isTargetOptions() {} + +func NewGenericS3Target(options *GenericS3TargetOptions) *Target { + return newTarget("org.osbuild.generic.s3", options) +} + +type GenericS3TargetResultOptions AWSS3TargetResultOptions + +func (GenericS3TargetResultOptions) isTargetResultOptions() {} + +func NewGenericS3TargetResult(options *GenericS3TargetResultOptions) *TargetResult { + return newTargetResult("org.osbuild.generic.s3", options) +} diff --git a/internal/target/target.go b/internal/target/target.go index fccc4c696..ace383c8a 100644 --- a/internal/target/target.go +++ b/internal/target/target.go @@ -83,6 +83,8 @@ func UnmarshalTargetOptions(targetName string, rawOptions json.RawMessage) (Targ options = new(VMWareTargetOptions) case "org.osbuild.oci": options = new(OCITargetOptions) + case "org.osbuild.generic.s3": + options = new(GenericS3TargetOptions) default: return nil, errors.New("unexpected target name") } diff --git a/internal/target/targetresult.go b/internal/target/targetresult.go index 737c93ecd..e3b29277c 100644 --- a/internal/target/targetresult.go +++ b/internal/target/targetresult.go @@ -55,8 +55,10 @@ func UnmarshalTargetResultOptions(trName string, rawOptions json.RawMessage) (Ta options = new(AzureImageTargetResultOptions) case "org.osbuild.oci": options = new(OCITargetResultOptions) + case "org.osbuild.generic.s3": + options = new(GenericS3TargetResultOptions) default: - return nil, fmt.Errorf("Unexpected target result name: %s", trName) + return nil, fmt.Errorf("unexpected target result name: %s", trName) } err := json.Unmarshal(rawOptions, options) diff --git a/internal/weldr/upload.go b/internal/weldr/upload.go index 226dbe05e..84a51ac64 100644 --- a/internal/weldr/upload.go +++ b/internal/weldr/upload.go @@ -36,6 +36,17 @@ type awsUploadSettings struct { func (awsUploadSettings) isUploadSettings() {} +type awsS3UploadSettings struct { + Region string `json:"region"` + AccessKeyID string `json:"accessKeyID,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty"` + SessionToken string `json:"sessionToken,omitempty"` + Bucket string `json:"bucket"` + Key string `json:"key"` +} + +func (awsS3UploadSettings) isUploadSettings() {} + type azureUploadSettings struct { StorageAccount string `json:"storageAccount,omitempty"` StorageAccessKey string `json:"storageAccessKey,omitempty"` @@ -68,6 +79,13 @@ type ociUploadSettings struct { func (ociUploadSettings) isUploadSettings() {} +type genericS3UploadSettings struct { + awsS3UploadSettings + Endpoint string `json:"endpoint"` +} + +func (genericS3UploadSettings) isUploadSettings() {} + type uploadRequest struct { Provider string `json:"provider"` ImageName string `json:"image_name"` @@ -93,10 +111,14 @@ func (u *uploadRequest) UnmarshalJSON(data []byte) error { settings = new(azureUploadSettings) case "aws": settings = new(awsUploadSettings) + case "aws.s3": + settings = new(awsS3UploadSettings) case "vmware": settings = new(vmwareUploadSettings) case "oci": settings = new(ociUploadSettings) + case "generic.s3": + settings = new(genericS3UploadSettings) default: return errors.New("unexpected provider name") } @@ -167,6 +189,27 @@ func targetsToUploadResponses(targets []*target.Target, state ComposeState) []up // Username and Password are intentionally not included. } uploads = append(uploads, upload) + case *target.AWSS3TargetOptions: + upload.ProviderName = "aws.s3" + upload.Settings = &awsS3UploadSettings{ + Region: options.Region, + Bucket: options.Bucket, + Key: options.Key, + // AccessKeyID and SecretAccessKey are intentionally not included. + } + uploads = append(uploads, upload) + case *target.GenericS3TargetOptions: + upload.ProviderName = "generic.s3" + upload.Settings = &genericS3UploadSettings{ + awsS3UploadSettings: awsS3UploadSettings{ + Region: options.Region, + Bucket: options.Bucket, + Key: options.Key, + // AccessKeyID and SecretAccessKey are intentionally not included. + }, + Endpoint: options.Endpoint, + } + uploads = append(uploads, upload) } } @@ -193,6 +236,17 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target. Bucket: options.Bucket, Key: options.Key, } + case *awsS3UploadSettings: + t.Name = "org.osbuild.aws.s3" + t.Options = &target.AWSS3TargetOptions{ + Filename: imageType.Filename(), + Region: options.Region, + AccessKeyID: options.AccessKeyID, + SecretAccessKey: options.SecretAccessKey, + SessionToken: options.SessionToken, + Bucket: options.Bucket, + Key: options.Key, + } case *azureUploadSettings: t.Name = "org.osbuild.azure" t.Options = &target.AzureTargetOptions{ @@ -225,6 +279,20 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target. Namespace: options.Namespace, Compartment: options.Compartment, } + case *genericS3UploadSettings: + t.Name = "org.osbuild.generic.s3" + t.Options = &target.GenericS3TargetOptions{ + AWSS3TargetOptions: target.AWSS3TargetOptions{ + Filename: imageType.Filename(), + Region: options.Region, + AccessKeyID: options.AccessKeyID, + SecretAccessKey: options.SecretAccessKey, + SessionToken: options.SessionToken, + Bucket: options.Bucket, + Key: options.Key, + }, + Endpoint: options.Endpoint, + } } return &t diff --git a/osbuild-composer.spec b/osbuild-composer.spec index a6b024cb4..da31543aa 100644 --- a/osbuild-composer.spec +++ b/osbuild-composer.spec @@ -228,6 +228,7 @@ install -m 0755 -vp tools/run-koji-container.sh %{buildroot}% install -m 0755 -vp tools/koji-compose.py %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/koji-compose-v2.py %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/libvirt_test.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/ +install -m 0755 -vp tools/s3_test.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/set-env-variables.sh %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vp tools/test-case-generators/generate-test-cases %{buildroot}%{_libexecdir}/osbuild-composer-test/ install -m 0755 -vd %{buildroot}%{_libexecdir}/tests/osbuild-composer diff --git a/test/cases/aws_s3.sh b/test/cases/aws_s3.sh new file mode 100755 index 000000000..82dd9ab09 --- /dev/null +++ b/test/cases/aws_s3.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -euo pipefail + +source /usr/libexec/osbuild-composer-test/set-env-variables.sh + +# Container image used for cloud provider CLI tools +CONTAINER_IMAGE_CLOUD_TOOLS="quay.io/osbuild/cloud-tools:latest" + +# Provision the software under test. +/usr/libexec/osbuild-composer-test/provision.sh + +# Check available container runtime +if which podman 2>/dev/null >&2; then + CONTAINER_RUNTIME=podman +elif which docker 2>/dev/null >&2; then + CONTAINER_RUNTIME=docker +else + echo No container runtime found, install podman or docker. + exit 2 +fi + +TEMPDIR=$(mktemp -d) +function cleanup() { + sudo rm -rf "$TEMPDIR" +} +trap cleanup EXIT + +# Generate a string, which can be used as a predictable resource name, +# especially when running the test in CI where we may need to clean up +# resources in case the test unexpectedly fails or is canceled +CI="${CI:-false}" +if [[ "$CI" == true ]]; then + # in CI, imitate GenerateCIArtifactName() from internal/test/helpers.go + TEST_ID="$DISTRO_CODE-$ARCH-$CI_COMMIT_BRANCH-$CI_BUILD_ID" +else + # if not running in Jenkins, generate ID not relying on specific env variables + TEST_ID=$(uuidgen); +fi + +# Set up temporary files. +AWS_S3_PROVIDER_CONFIG=${TEMPDIR}/aws.toml + +# We need awscli to talk to AWS. +if ! hash aws; then + echo "Using 'awscli' from a container" + sudo ${CONTAINER_RUNTIME} pull ${CONTAINER_IMAGE_CLOUD_TOOLS} + + AWS_CMD="sudo ${CONTAINER_RUNTIME} run --rm \ + -e AWS_ACCESS_KEY_ID=${V2_AWS_ACCESS_KEY_ID} \ + -e AWS_SECRET_ACCESS_KEY=${V2_AWS_SECRET_ACCESS_KEY} \ + ${CONTAINER_IMAGE_CLOUD_TOOLS} aws --region $AWS_REGION" +else + echo "Using pre-installed 'aws' from the system" + AWS_CMD="aws --region $AWS_REGION" +fi +$AWS_CMD --version + +# Write an AWS TOML file +tee "$AWS_S3_PROVIDER_CONFIG" > /dev/null << EOF +provider = "aws.s3" + +[settings] +accessKeyID = "${V2_AWS_ACCESS_KEY_ID}" +secretAccessKey = "${V2_AWS_SECRET_ACCESS_KEY}" +bucket = "${AWS_BUCKET}" +region = "${AWS_REGION}" +key = "${TEST_ID}" +EOF + +IMAGE_OBJECT_KEY="${AWS_BUCKET}/${TEST_ID}-disk.qcow2" + +/usr/libexec/osbuild-composer-test/s3_test.sh "${TEST_ID}" "${AWS_S3_PROVIDER_CONFIG}" "${AWS_CMD} s3 ls ${IMAGE_OBJECT_KEY}" "${AWS_CMD} s3 presign ${IMAGE_OBJECT_KEY}" "${AWS_CMD} s3 rm s3://${IMAGE_OBJECT_KEY}" diff --git a/test/cases/generic_s3.sh b/test/cases/generic_s3.sh new file mode 100755 index 000000000..48b3c0f0a --- /dev/null +++ b/test/cases/generic_s3.sh @@ -0,0 +1,98 @@ +#!/bin/bash +set -euo pipefail + +source /usr/libexec/osbuild-composer-test/set-env-variables.sh + +# Container images for MinIO Server and Client +CONTAINER_MINIO_CLIENT="quay.io/minio/mc:latest" +CONTAINER_MINIO_SERVER="quay.io/minio/minio:latest" + +# Provision the software under test. +/usr/libexec/osbuild-composer-test/provision.sh + +# Check available container runtime +if which podman 2>/dev/null >&2; then + CONTAINER_RUNTIME=podman +elif which docker 2>/dev/null >&2; then + CONTAINER_RUNTIME=docker +else + echo No container runtime found, install podman or docker. + exit 2 +fi + +TEMPDIR=$(mktemp -d) +function cleanup() { + sudo rm -rf "$TEMPDIR" +} +trap cleanup EXIT + +# Generate a string, which can be used as a predictable resource name, +# especially when running the test in CI where we may need to clean up +# resources in case the test unexpectedly fails or is canceled +CI="${CI:-false}" +if [[ "$CI" == true ]]; then + # in CI, imitate GenerateCIArtifactName() from internal/test/helpers.go + TEST_ID="$DISTRO_CODE-$ARCH-$CI_COMMIT_BRANCH-$CI_BUILD_ID" +else + # if not running in Jenkins, generate ID not relying on specific env variables + TEST_ID=$(uuidgen); +fi + +# Set up temporary files. +MINIO_CONFIG_DIR=${TEMPDIR}/minio-config +MINIO_PROVIDER_CONFIG=${TEMPDIR}/minio.toml + +# We need MinIO Client to talk to the MinIO Server. +if ! hash mc; then + echo "Using 'mc' from a container" + sudo ${CONTAINER_RUNTIME} pull ${CONTAINER_MINIO_CLIENT} + + MC_CMD="sudo ${CONTAINER_RUNTIME} run --rm \ + -v ${MINIO_CONFIG_DIR}:${MINIO_CONFIG_DIR}:Z \ + --network=host \ + ${CONTAINER_MINIO_CLIENT} --config-dir=${MINIO_CONFIG_DIR}" +else + echo "Using pre-installed 'mc' from the system" + MC_CMD="mc --config-dir=${MINIO_CONFIG_DIR}" +fi +mkdir "${MINIO_CONFIG_DIR}" +$MC_CMD --version + +MINIO_CONTAINER_NAME="minio-server" +MINIO_ENDPOINT="http://localhost:9000" +MINIO_ROOT_USER="X29DU5Q6C5NKDQ8PLGVT" +MINIO_ROOT_PASSWORD=$(date +%s | sha256sum | base64 | head -c 32 ; echo) +MINIO_SERVER_ALIAS=local +MINIO_BUCKET="ci-test" +MINIO_REGION="us-east-1" +# Write an AWS TOML file +tee "$MINIO_PROVIDER_CONFIG" > /dev/null << EOF +provider = "generic.s3" + +[settings] +endpoint = "${MINIO_ENDPOINT}" +accessKeyID = "${MINIO_ROOT_USER}" +secretAccessKey = "${MINIO_ROOT_PASSWORD}" +bucket = "${MINIO_BUCKET}" +region = "${MINIO_REGION}" +key = "${TEST_ID}" +EOF + +# Start the MinIO Server +${CONTAINER_RUNTIME} run --rm -d \ + --name ${MINIO_CONTAINER_NAME} \ + -p 9000:9000 \ + -e MINIO_BROWSER=off \ + -e MINIO_ROOT_USER="${MINIO_ROOT_USER}" \ + -e MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD}" \ + ${CONTAINER_MINIO_SERVER} server /data +# Kill the server once we're done +trap '${CONTAINER_RUNTIME} kill ${MINIO_CONTAINER_NAME}' EXIT + +# Configure the local server +${MC_CMD} alias set ${MINIO_SERVER_ALIAS} ${MINIO_ENDPOINT} ${MINIO_ROOT_USER} "${MINIO_ROOT_PASSWORD}" +# Create the bucket +${MC_CMD} mb ${MINIO_SERVER_ALIAS}/${MINIO_BUCKET} + +IMAGE_OBJECT_KEY="${MINIO_SERVER_ALIAS}/${MINIO_BUCKET}/${TEST_ID}-disk.qcow2" +/usr/libexec/osbuild-composer-test/s3_test.sh "${TEST_ID}" "${MINIO_PROVIDER_CONFIG}" "${MC_CMD} ls ${IMAGE_OBJECT_KEY}" "${MC_CMD} --json share download ${IMAGE_OBJECT_KEY} | jq .share | tr -d '\"'" diff --git a/tools/libvirt_test.sh b/tools/libvirt_test.sh index 1cbdc7f6d..ef95aba92 100755 --- a/tools/libvirt_test.sh +++ b/tools/libvirt_test.sh @@ -17,6 +17,8 @@ IMAGE_TYPE=${1:-qcow2} # Take the boot type passed to the script or use BIOS by default if nothing # was passed. BOOT_TYPE=${2:-bios} +# Take the image from the url passes to the script or build it by default if nothing +LIBVIRT_IMAGE_URL=${3:-""} # Select the file extension based on the image that we are building. IMAGE_EXTENSION=$IMAGE_TYPE @@ -131,82 +133,90 @@ get_compose_metadata () { sudo cat "${COMPOSE_ID}".json | jq -M '.' | tee "$METADATA_FILE" > /dev/null } -# Write a basic blueprint for our image. -tee "$BLUEPRINT_FILE" > /dev/null << EOF -name = "bp" -description = "A base system" -version = "0.0.1" -EOF - -# Prepare the blueprint for the compose. -greenprint "📋 Preparing blueprint" -sudo composer-cli blueprints push "$BLUEPRINT_FILE" -sudo composer-cli blueprints depsolve bp - -# Get worker unit file so we can watch the journal. -WORKER_UNIT=$(sudo systemctl list-units | grep -o -E "osbuild.*worker.*\.service") -sudo journalctl -af -n 1 -u "${WORKER_UNIT}" & -WORKER_JOURNAL_PID=$! -# Stop watching the worker journal when exiting. -trap 'sudo pkill -P ${WORKER_JOURNAL_PID}' EXIT - -# Start the compose -greenprint "🚀 Starting compose" -sudo composer-cli --json compose start bp "$IMAGE_TYPE" | tee "$COMPOSE_START" -if rpm -q --quiet weldr-client; then - COMPOSE_ID=$(jq -r '.body.build_id' "$COMPOSE_START") -else - COMPOSE_ID=$(jq -r '.build_id' "$COMPOSE_START") -fi - -# Wait for the compose to finish. -greenprint "⏱ Waiting for compose to finish: ${COMPOSE_ID}" -while true; do - sudo composer-cli --json compose info "${COMPOSE_ID}" | tee "$COMPOSE_INFO" > /dev/null - if rpm -q --quiet weldr-client; then - COMPOSE_STATUS=$(jq -r '.body.queue_status' "$COMPOSE_INFO") - else - COMPOSE_STATUS=$(jq -r '.queue_status' "$COMPOSE_INFO") - fi - - # Is the compose finished? - if [[ $COMPOSE_STATUS != RUNNING ]] && [[ $COMPOSE_STATUS != WAITING ]]; then - break - fi - - # Wait 30 seconds and try again. - sleep 5 -done - -# Capture the compose logs from osbuild. -greenprint "💬 Getting compose log and metadata" -get_compose_log "$COMPOSE_ID" -get_compose_metadata "$COMPOSE_ID" - -# Kill the journal monitor immediately and remove the trap -sudo pkill -P ${WORKER_JOURNAL_PID} -trap - EXIT - -# Did the compose finish with success? -if [[ $COMPOSE_STATUS != FINISHED ]]; then - echo "Something went wrong with the compose. 😢" - exit 1 -fi - -# Download the image. -greenprint "📥 Downloading the image" - # Current $PWD is inside /tmp, there may not be enough space for an image. # Let's use a bigger temporary directory for this operation. BIG_TEMP_DIR=/var/lib/osbuild-composer-tests sudo rm -rf "${BIG_TEMP_DIR}" || true sudo mkdir "${BIG_TEMP_DIR}" -pushd "${BIG_TEMP_DIR}" - sudo composer-cli compose image "${COMPOSE_ID}" > /dev/null - IMAGE_FILENAME=$(basename "$(find . -maxdepth 1 -type f -name "*.${IMAGE_EXTENSION}")") + +if [ -z "${LIBVIRT_IMAGE_URL}" ]; then + # Write a basic blueprint for our image. + tee "$BLUEPRINT_FILE" > /dev/null << EOF +name = "bp" +description = "A base system" +version = "0.0.1" +EOF + + # Prepare the blueprint for the compose. + greenprint "📋 Preparing blueprint" + sudo composer-cli blueprints push "$BLUEPRINT_FILE" + sudo composer-cli blueprints depsolve bp + + # Get worker unit file so we can watch the journal. + WORKER_UNIT=$(sudo systemctl list-units | grep -o -E "osbuild.*worker.*\.service") + sudo journalctl -af -n 1 -u "${WORKER_UNIT}" & + WORKER_JOURNAL_PID=$! + # Stop watching the worker journal when exiting. + trap 'sudo pkill -P ${WORKER_JOURNAL_PID}' EXIT + + # Start the compose + greenprint "🚀 Starting compose" + sudo composer-cli --json compose start bp "$IMAGE_TYPE" | tee "$COMPOSE_START" + if rpm -q --quiet weldr-client; then + COMPOSE_ID=$(jq -r '.body.build_id' "$COMPOSE_START") + else + COMPOSE_ID=$(jq -r '.build_id' "$COMPOSE_START") + fi + + # Wait for the compose to finish. + greenprint "⏱ Waiting for compose to finish: ${COMPOSE_ID}" + while true; do + sudo composer-cli --json compose info "${COMPOSE_ID}" | tee "$COMPOSE_INFO" > /dev/null + if rpm -q --quiet weldr-client; then + COMPOSE_STATUS=$(jq -r '.body.queue_status' "$COMPOSE_INFO") + else + COMPOSE_STATUS=$(jq -r '.queue_status' "$COMPOSE_INFO") + fi + + # Is the compose finished? + if [[ $COMPOSE_STATUS != RUNNING ]] && [[ $COMPOSE_STATUS != WAITING ]]; then + break + fi + + # Wait 30 seconds and try again. + sleep 5 + done + + # Capture the compose logs from osbuild. + greenprint "💬 Getting compose log and metadata" + get_compose_log "$COMPOSE_ID" + get_compose_metadata "$COMPOSE_ID" + + # Kill the journal monitor immediately and remove the trap + sudo pkill -P ${WORKER_JOURNAL_PID} + trap - EXIT + + # Did the compose finish with success? + if [[ $COMPOSE_STATUS != FINISHED ]]; then + echo "Something went wrong with the compose. 😢" + exit 1 + fi + + # Download the image. + greenprint "📥 Downloading the image" + + pushd "${BIG_TEMP_DIR}" + sudo composer-cli compose image "${COMPOSE_ID}" > /dev/null + IMAGE_FILENAME=$(basename "$(find . -maxdepth 1 -type f -name "*.${IMAGE_EXTENSION}")") + LIBVIRT_IMAGE_PATH=/var/lib/libvirt/images/${IMAGE_KEY}.${IMAGE_EXTENSION} + sudo mv "$IMAGE_FILENAME" "$LIBVIRT_IMAGE_PATH" + popd +else + pushd "${BIG_TEMP_DIR}" LIBVIRT_IMAGE_PATH=/var/lib/libvirt/images/${IMAGE_KEY}.${IMAGE_EXTENSION} - sudo mv "$IMAGE_FILENAME" "$LIBVIRT_IMAGE_PATH" -popd + sudo curl -o "${LIBVIRT_IMAGE_PATH}" "${LIBVIRT_IMAGE_URL}" + popd +fi # Prepare cloud-init data. CLOUD_INIT_DIR=$(mktemp -d) @@ -323,8 +333,10 @@ else fi sudo rm -f "$LIBVIRT_IMAGE_PATH" $CLOUD_INIT_PATH -# Also delete the compose so we don't run out of disk space -sudo composer-cli compose delete "${COMPOSE_ID}" > /dev/null +if [ -z "${LIBVIRT_IMAGE_URL}" ]; then + # Also delete the compose so we don't run out of disk space + sudo composer-cli compose delete "${COMPOSE_ID}" > /dev/null +fi # Use the return code of the smoke test to determine if we passed or failed. if [[ $RESULTS == 1 ]]; then diff --git a/tools/s3_test.sh b/tools/s3_test.sh new file mode 100755 index 000000000..34bd69bca --- /dev/null +++ b/tools/s3_test.sh @@ -0,0 +1,145 @@ +#!/bin/bash +set -euo pipefail + +source /usr/libexec/osbuild-composer-test/set-env-variables.sh + +TEST_ID=${1} +S3_PROVIDER_CONFIG_FILE=${2} +S3_CHECK_CMD=${3} +S3_GET_URL_CMD=${4} +S3_DELETE_CMD=${5:-""} + +# Colorful output. +function greenprint { + echo -e "\033[1;32m[$(date -Isecond)] ${1}\033[0m" +} + +function get_build_info() { + key="$1" + fname="$2" + if rpm -q --quiet weldr-client; then + key=".body${key}" + fi + jq -r "${key}" "${fname}" +} + +TEMPDIR=$(mktemp -d) +function cleanup() { + sudo rm -rf "$TEMPDIR" +} +trap cleanup EXIT + +# Jenkins sets WORKSPACE to the job workspace, but if this script runs +# outside of Jenkins, we can set up a temporary directory instead. +if [[ ${WORKSPACE:-empty} == empty ]]; then + WORKSPACE=$(mktemp -d) +fi + +# Set up temporary files. +BLUEPRINT_FILE=${TEMPDIR}/blueprint.toml +BLUEPRINT_NAME=empty +COMPOSE_START=${TEMPDIR}/compose-start-${TEST_ID}.json +COMPOSE_INFO=${TEMPDIR}/compose-info-${TEST_ID}.json + +# Get the compose log. +get_compose_log () { + COMPOSE_ID=$1 + LOG_FILE=${WORKSPACE}/osbuild-${ID}-${VERSION_ID}-aws.log + + # Download the logs. + sudo composer-cli compose log "$COMPOSE_ID" | tee "$LOG_FILE" > /dev/null +} + +# Get the compose metadata. +get_compose_metadata () { + COMPOSE_ID=$1 + METADATA_FILE=${WORKSPACE}/osbuild-${ID}-${VERSION_ID}-aws.json + + # Download the metadata. + sudo composer-cli compose metadata "$COMPOSE_ID" > /dev/null + + # Find the tarball and extract it. + TARBALL=$(basename "$(find . -maxdepth 1 -type f -name "*-metadata.tar")") + sudo tar -xf "$TARBALL" + sudo rm -f "$TARBALL" + + # Move the JSON file into place. + sudo cat "${COMPOSE_ID}".json | jq -M '.' | tee "$METADATA_FILE" > /dev/null +} + +# Write a basic blueprint for our image. +tee "$BLUEPRINT_FILE" > /dev/null << EOF +name = "${BLUEPRINT_NAME}" +description = "A base system with bash" +version = "0.0.1" +EOF + +# Prepare the blueprint for the compose. +greenprint "📋 Preparing blueprint" +sudo composer-cli blueprints push "$BLUEPRINT_FILE" +sudo composer-cli blueprints depsolve ${BLUEPRINT_NAME} + +# Get worker unit file so we can watch the journal. +WORKER_UNIT=$(sudo systemctl list-units | grep -o -E "osbuild.*worker.*\.service") +sudo journalctl -af -n 1 -u "${WORKER_UNIT}" & +WORKER_JOURNAL_PID=$! +# Stop watching the worker journal when exiting. +trap 'sudo pkill -P ${WORKER_JOURNAL_PID}' EXIT + +# Start the compose and upload to AWS. +greenprint "🚀 Starting compose" +sudo composer-cli --json compose start ${BLUEPRINT_NAME} qcow2 "$TEST_ID" "$S3_PROVIDER_CONFIG_FILE" | tee "$COMPOSE_START" +COMPOSE_ID=$(get_build_info ".build_id" "$COMPOSE_START") + +# Wait for the compose to finish. +greenprint "⏱ Waiting for compose to finish: ${COMPOSE_ID}" +while true; do + sudo composer-cli --json compose info "${COMPOSE_ID}" | tee "$COMPOSE_INFO" > /dev/null + COMPOSE_STATUS=$(get_build_info ".queue_status" "$COMPOSE_INFO") + + # Is the compose finished? + if [[ $COMPOSE_STATUS != RUNNING ]] && [[ $COMPOSE_STATUS != WAITING ]]; then + break + fi + + # Wait 30 seconds and try again. + sleep 30 +done + +# Capture the compose logs from osbuild. +greenprint "💬 Getting compose log and metadata" +get_compose_log "$COMPOSE_ID" +get_compose_metadata "$COMPOSE_ID" + +# Kill the journal monitor immediately and remove the trap +sudo pkill -P ${WORKER_JOURNAL_PID} +trap - EXIT + +# Did the compose finish with success? +if [[ $COMPOSE_STATUS != FINISHED ]]; then + echo "Something went wrong with the compose. 😢" + exit 1 +fi + +# Delete the compose so we don't run out of disk space +sudo composer-cli compose delete "${COMPOSE_ID}" > /dev/null + +# Find the image that we made in the AWS Bucket +greenprint "🔍 Search for created image" +if ! bash -c "${S3_CHECK_CMD}"; then + echo "Failed to find the image in the S3 Bucket" + exit 1 +fi + +function removeImageFromS3() { + bash -c "${S3_DELETE_CMD}" +} +if [ -n "${S3_DELETE_CMD}" ]; then + trap removeImageFromS3 EXIT +fi + +# Generate a URL for the image +QCOW2_IMAGE_URL=$(bash -c "${S3_GET_URL_CMD}") + +# Run the image on KVM +/usr/libexec/osbuild-composer-test/libvirt_test.sh qcow2 bios "${QCOW2_IMAGE_URL}"