From a632848c59dd432d8a930e4aa4db44b0137991f8 Mon Sep 17 00:00:00 2001 From: Sanne Raymaekers Date: Wed, 6 Sep 2023 13:49:05 +0200 Subject: [PATCH] cmd/osbuild-worker: add default OCI configuration Useful for hosted deployments, where target options are often empty or incomplete. --- cmd/osbuild-worker/config.go | 5 ++ cmd/osbuild-worker/config_test.go | 6 +++ cmd/osbuild-worker/jobimpl-osbuild.go | 75 ++++++++++++++++++++++----- cmd/osbuild-worker/main.go | 32 ++++++++++++ internal/upload/oci/upload.go | 10 ++-- 5 files changed, 111 insertions(+), 17 deletions(-) diff --git a/cmd/osbuild-worker/config.go b/cmd/osbuild-worker/config.go index 2f0d9b76a..43491d0cc 100644 --- a/cmd/osbuild-worker/config.go +++ b/cmd/osbuild-worker/config.go @@ -38,6 +38,10 @@ type awsConfig struct { Bucket string `toml:"bucket"` } +type ociConfig struct { + Credentials string `toml:"credentials"` +} + type genericS3Config struct { Credentials string `toml:"credentials"` Endpoint string `toml:"endpoint"` @@ -71,6 +75,7 @@ type workerConfig struct { GenericS3 *genericS3Config `toml:"generic_s3"` Authentication *authenticationConfig `toml:"authentication"` Containers *containersConfig `toml:"containers"` + OCI *ociConfig `toml:"oci"` // default value: /api/worker/v1 BasePath string `toml:"base_path"` DNFJson string `toml:"dnf-json"` diff --git a/cmd/osbuild-worker/config_test.go b/cmd/osbuild-worker/config_test.go index aa2cb33b7..34b5115c5 100644 --- a/cmd/osbuild-worker/config_test.go +++ b/cmd/osbuild-worker/config_test.go @@ -49,6 +49,9 @@ upload_threads = 8 credentials = "/etc/osbuild-worker/aws-creds" bucket = "buckethead" +[oci] +credentials = "/etc/osbuild-worker/oci-creds" + [generic_s3] credentials = "/etc/osbuild-worker/s3-creds" endpoint = "http://s3.example.com" @@ -96,6 +99,9 @@ offline_token = "/etc/osbuild-worker/offline_token" Credentials: "/etc/osbuild-worker/aws-creds", Bucket: "buckethead", }, + OCI: &ociConfig{ + Credentials: "/etc/osbuild-worker/oci-creds", + }, GenericS3: &genericS3Config{ Credentials: "/etc/osbuild-worker/s3-creds", Endpoint: "http://s3.example.com", diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index d68ada731..3625fc0b8 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -58,12 +58,20 @@ type AzureConfiguration struct { UploadThreads int } +type OCIConfiguration struct { + ClientParams *oci.ClientParams + Compartment string + Bucket string + Namespace string +} + type OSBuildJobImpl struct { Store string Output string KojiServers map[string]kojiServer GCPConfig GCPConfiguration AzureConfig AzureConfiguration + OCIConfig OCIConfiguration AWSCreds string AWSBucket string S3Config S3Configuration @@ -179,6 +187,30 @@ func (impl *OSBuildJobImpl) getGCP(credentials []byte) (*gcp.GCP, error) { } } +// Takes the worker config as a base and overwrites it with both t1 and t2's options +func (impl *OSBuildJobImpl) getOCI(tcp oci.ClientParams) (oci.Client, error) { + var cp oci.ClientParams + if impl.OCIConfig.ClientParams != nil { + cp = *impl.OCIConfig.ClientParams + } + if tcp.User != "" { + cp.User = tcp.User + } + if tcp.Region != "" { + cp.Region = tcp.Region + } + if tcp.Tenancy != "" { + cp.Tenancy = tcp.Tenancy + } + if tcp.PrivateKey != "" { + cp.PrivateKey = tcp.PrivateKey + } + if tcp.Fingerprint != "" { + cp.Fingerprint = tcp.Fingerprint + } + return oci.NewClient(&cp) +} + func validateResult(result *worker.OSBuildJobResult, jobID string) { logWithId := logrus.WithField("jobId", jobID) if result.JobError != nil { @@ -848,7 +880,7 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { targetResult = target.NewOCITargetResult(nil) // create an ociClient uploader with a valid storage client var ociClient oci.Client - ociClient, err = oci.NewClient(&oci.ClientParams{ + ociClient, err = impl.getOCI(oci.ClientParams{ User: targetOptions.User, Region: targetOptions.Region, Tenancy: targetOptions.Tenancy, @@ -868,10 +900,18 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { } defer file.Close() i, _ := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) + bucket := impl.OCIConfig.Bucket + if targetOptions.Bucket != "" { + bucket = targetOptions.Bucket + } + namespace := impl.OCIConfig.Namespace + if targetOptions.Namespace != "" { + namespace = targetOptions.Namespace + } err = ociClient.Upload( fmt.Sprintf("osbuild-upload-%d", i), - targetOptions.Bucket, - targetOptions.Namespace, + bucket, + namespace, file, ) if err != nil { @@ -879,11 +919,15 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { break } + compartment := impl.OCIConfig.Compartment + if targetOptions.Compartment != "" { + compartment = targetOptions.Compartment + } imageID, err := ociClient.CreateImage( fmt.Sprintf("osbuild-upload-%d", i), - targetOptions.Bucket, - targetOptions.Namespace, - targetOptions.Compartment, + bucket, + namespace, + compartment, jobTarget.ImageName, ) if err != nil { @@ -896,8 +940,7 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { case *target.OCIObjectStorageTargetOptions: targetResult = target.NewOCIObjectStorageTargetResult(nil) // create an ociClient uploader with a valid storage client - var ociClient oci.Client - ociClient, err = oci.NewClient(&oci.ClientParams{ + ociClient, err := impl.getOCI(oci.ClientParams{ User: targetOptions.User, Region: targetOptions.Region, Tenancy: targetOptions.Tenancy, @@ -917,10 +960,18 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { } defer file.Close() i, _ := rand.Int(rand.Reader, big.NewInt(math.MaxInt64)) + bucket := impl.OCIConfig.Bucket + if targetOptions.Bucket != "" { + bucket = targetOptions.Bucket + } + namespace := impl.OCIConfig.Namespace + if targetOptions.Namespace != "" { + namespace = targetOptions.Namespace + } err = ociClient.Upload( fmt.Sprintf("osbuild-upload-%d", i), - targetOptions.Bucket, - targetOptions.Namespace, + bucket, + namespace, file, ) if err != nil { @@ -928,12 +979,12 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { break } - uri, err := ociClient.PreAuthenticatedRequest(fmt.Sprintf("osbuild-upload-%d", i), targetOptions.Bucket, targetOptions.Namespace) + uri, err := ociClient.PreAuthenticatedRequest(fmt.Sprintf("osbuild-upload-%d", i), bucket, namespace) if err != nil { targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorGeneratingSignedURL, err.Error(), nil) break } - logWithId.Info("[OCI] 🎉 Image uploaded and registered!") + logWithId.Info("[OCI] 🎉 Image uploaded and pre-authenticated request generated!") targetResult.Options = &target.OCIObjectStorageTargetResultOptions{URL: uri} case *target.ContainerTargetOptions: targetResult = target.NewContainerTargetResult(nil) diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go index bc0159e52..3c5a7b70e 100644 --- a/cmd/osbuild-worker/main.go +++ b/cmd/osbuild-worker/main.go @@ -24,6 +24,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/dnfjson" "github.com/osbuild/osbuild-composer/internal/upload/azure" "github.com/osbuild/osbuild-composer/internal/upload/koji" + "github.com/osbuild/osbuild-composer/internal/upload/oci" "github.com/osbuild/osbuild-composer/internal/worker" ) @@ -395,6 +396,36 @@ func main() { containersTLSVerify = config.Containers.TLSVerify } + var ociConfig OCIConfiguration + if config.OCI != nil { + var creds struct { + User string `toml:"user"` + Tenancy string `toml:"tenancy"` + Region string `toml:"region"` + Fingerprint string `toml:"fingerprint"` + PrivateKey string `toml:"private_key"` + Bucket string `toml:"bucket"` + Namespace string `toml:"namespace"` + Compartment string `toml:"compartment"` + } + _, err := toml.DecodeFile(config.OCI.Credentials, &creds) + if err != nil { + logrus.Fatalf("cannot load oci credentials: %v", err) + } + ociConfig = OCIConfiguration{ + ClientParams: &oci.ClientParams{ + User: creds.User, + Region: creds.Region, + Tenancy: creds.Tenancy, + PrivateKey: creds.PrivateKey, + Fingerprint: creds.Fingerprint, + }, + Bucket: creds.Bucket, + Namespace: creds.Namespace, + Compartment: creds.Compartment, + } + } + // depsolve jobs can be done during other jobs depsolveCtx, depsolveCtxCancel := context.WithCancel(context.Background()) solver := dnfjson.NewBaseSolver(rpmmd_cache) @@ -438,6 +469,7 @@ func main() { KojiServers: kojiServers, GCPConfig: gcpConfig, AzureConfig: azureConfig, + OCIConfig: ociConfig, AWSCreds: awsCredentials, AWSBucket: awsBucket, S3Config: S3Configuration{ diff --git a/internal/upload/oci/upload.go b/internal/upload/oci/upload.go index 6b4c31a52..bfd07f5bc 100644 --- a/internal/upload/oci/upload.go +++ b/internal/upload/oci/upload.go @@ -238,11 +238,11 @@ func (c Client) createImage(objectName, bucketName, namespace, compartmentID, im } type ClientParams struct { - User string - Region string - Tenancy string - PrivateKey string - Fingerprint string + User string `toml:"user"` + Region string `toml:"region"` + Tenancy string `toml:"tenancy"` + PrivateKey string `toml:"private_key"` + Fingerprint string `toml:"fingerprint"` } type ociClient struct {