From e698080bc7b4ceb52b3aaf7ac517675434bb046e Mon Sep 17 00:00:00 2001 From: Tomas Hozza Date: Thu, 11 Mar 2021 16:31:51 +0100 Subject: [PATCH] GCP: Set image name as custom metadata on uploaded image object Extend StorageObjectUpload() to allow setting custom metadata on the uploaded object. Modify worker's osbuild job implementation and GCP CLI upload tool to set the chosen image name as a custom metadata on the uploaded object. This will make it possible to connect Storage objects to specific images. Add News entry about image name being added as metadata to uploaded GCP Storage object as part of worker job. Signed-off-by: Tomas Hozza --- cmd/osbuild-upload-gcp/main.go | 3 ++- cmd/osbuild-worker/jobimpl-osbuild.go | 3 ++- .../unreleased/worker-gcp-uploaded-object-meta.md | 6 ++++++ internal/cloud/gcp/storage.go | 14 +++++++++++++- 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 docs/news/unreleased/worker-gcp-uploaded-object-meta.md diff --git a/cmd/osbuild-upload-gcp/main.go b/cmd/osbuild-upload-gcp/main.go index 89da016d7..916894aec 100644 --- a/cmd/osbuild-upload-gcp/main.go +++ b/cmd/osbuild-upload-gcp/main.go @@ -63,7 +63,8 @@ func main() { // Upload image to the Storage if !skipUpload { log.Printf("[GCP] 🚀 Uploading image to: %s/%s", bucketName, objectName) - _, err := g.StorageObjectUpload(imageFile, bucketName, objectName) + _, err := g.StorageObjectUpload(imageFile, bucketName, objectName, + map[string]string{gcp.MetadataKeyImageName: imageName}) if err != nil { log.Fatalf("[GCP] Uploading image failed: %v", err) } diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index 3a332f80b..88c024fc4 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -268,7 +268,8 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { } log.Printf("[GCP] 🚀 Uploading image to: %s/%s", options.Bucket, options.Object) - _, err = g.StorageObjectUpload(path.Join(outputDirectory, options.Filename), options.Bucket, options.Object) + _, err = g.StorageObjectUpload(path.Join(outputDirectory, options.Filename), + options.Bucket, options.Object, map[string]string{gcp.MetadataKeyImageName: t.ImageName}) if err != nil { r = append(r, err) continue diff --git a/docs/news/unreleased/worker-gcp-uploaded-object-meta.md b/docs/news/unreleased/worker-gcp-uploaded-object-meta.md new file mode 100644 index 000000000..0dafe3ae2 --- /dev/null +++ b/docs/news/unreleased/worker-gcp-uploaded-object-meta.md @@ -0,0 +1,6 @@ +# Worker: Set image name as custom metadata on the file uploaded to GCP Storage + +Worker osbuild jobs with GCP upload target now set the chosen image name as +custom metadata on the uploaded object. This makes finding the uploaded +object using the image name possible. The behavior is useful mainly +for cleaning up cloud resources in case of unexpected failures. diff --git a/internal/cloud/gcp/storage.go b/internal/cloud/gcp/storage.go index 81e193c92..b75247b6b 100644 --- a/internal/cloud/gcp/storage.go +++ b/internal/cloud/gcp/storage.go @@ -15,6 +15,14 @@ import ( "google.golang.org/api/option" ) +const ( + // MetadataKeyImageName contains a key name used to store metadata on + // a Storage object with the intended name of the image. + // The metadata can be then used to associate the object with actual + // image build using the image name. + MetadataKeyImageName string = "osbuild-composer-image-name" +) + // StorageObjectUpload uploads an OS image to specified Cloud Storage bucket and object. // The bucket must exist. MD5 sum of the image file and uploaded object is // compared after the upload to verify the integrity of the uploaded image. @@ -23,7 +31,7 @@ import ( // // Uses: // - Storage API -func (g *GCP) StorageObjectUpload(filename, bucket, object string) (*storage.ObjectAttrs, error) { +func (g *GCP) StorageObjectUpload(filename, bucket, object string, metadata map[string]string) (*storage.ObjectAttrs, error) { ctx := context.Background() storageClient, err := storage.NewClient(ctx, option.WithCredentials(g.creds)) @@ -57,6 +65,10 @@ func (g *GCP) StorageObjectUpload(filename, bucket, object string) (*storage.Obj // Uploaded data is rejected if its MD5 hash does not match the set value. wc.MD5 = imageFileHash.Sum(nil) + if metadata != nil { + wc.ObjectAttrs.Metadata = metadata + } + if _, err = io.Copy(wc, imageFile); err != nil { return nil, fmt.Errorf("uploading the image failed: %v", err) }