particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

GCP: refactor logging and storage cleanup

Browse source 
Originally, the internal GCP library in `internal/upload/gcp` was
logging various information and errors. Refactor the code to move all
logging to callers of the library. As a result, some methods now return
additional information to preserve the same amount of information being
logged for GCP.

Refactor methods to have only single purpose and not do any extra work,
such as storage cleanup. Methods which create new resources now don't do
any cleanup at all. The caller is responsible to check for any errors
and perform any cleanup necessary. Necessary methods to perform cleanup
are provided.

Modify worker's job implementation and GCP CLI tool to explicitly do all
necessary cleanup, including in case of errors.

Signed-off-by: Tomas Hozza <thozza@redhat.com>
This commit is contained in:
Tomas Hozza 2021-03-08 15:57:48 +01:00 • committed by Ondřej Budai
parent d7b0323a2d
commit 7de2011beb
3 changed files with 238 additions and 180 deletions
Download patch file Download diff file Expand all files Collapse all files

50
cmd/osbuild-upload-gcp/main.go
View file

@ -4,7 +4,7 @@ import (
"flag"
"fmt"
"io/ioutil"
"os"
"log"
"github.com/osbuild/osbuild-composer/internal/upload/gcp"
)
@ -51,40 +51,60 @@ func main() {
var err error
credentials, err = ioutil.ReadFile(credentialsPath)
if err != nil {
fmt.Fprintf(os.Stderr, "Error while reading credentials: %s\n", err)
return
log.Fatalf("[GCP] Error while reading credentials: %v", err)
}
}
g, err := gcp.New(credentials)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
log.Fatalf("[GCP] Failed to create new GCP object: %v", err)
}
// Upload image to the Storage
if !skipUpload {
if err := g.Upload(imageFile, bucketName, objectName); err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
log.Printf("[GCP] 🚀 Uploading image to: %s/%s", bucketName, objectName)
_, err := g.StorageObjectUpload(imageFile, bucketName, objectName)
if err != nil {
log.Fatalf("[GCP] Uploading image failed: %v", err)
}
}
// Import Image to Compute Node
if !skipImport {
err = g.Import(bucketName, objectName, imageName, osFamily, region)
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
log.Printf("[GCP] 📥 Importing image into Compute Node as '%s'", imageName)
imageBuild, importErr := g.ComputeImageImport(bucketName, objectName, imageName, osFamily, region)
if imageBuild != nil {
log.Printf("[GCP] 📜 Image import log URL: %s", imageBuild.LogUrl)
log.Printf("[GCP] 🎉 Image import finished with status: %s", imageBuild.Status)
}
// Cleanup storage before checking for errors
log.Printf("[GCP] 🧹 Deleting uploaded image file: %s/%s", bucketName, objectName)
if err = g.StorageObjectDelete(bucketName, objectName); err != nil {
log.Printf("[GCP] Encountered error while deleting object: %v", err)
}
deleted, errs := g.StorageImageImportCleanup(imageName)
for _, d := range deleted {
log.Printf("[GCP] 🧹 Deleted image import job file '%s'", d)
}
for _, e := range errs {
log.Printf("[GCP] Encountered error during image import cleanup: %v", e)
}
// check error from ComputeImageImport()
if importErr != nil {
log.Fatalf("[GCP] Importing image failed: %v", err)
}
log.Printf("[GCP] 💿 Image URL: %s", g.ComputeImageURL(imageName))
}
// Share the imported Image with specified accounts using IAM policy
if len(shareWith) > 0 {
err = g.Share(imageName, []string(shareWith))
log.Printf("[GCP] 🔗 Sharing the image with: %+v", shareWith)
err = g.ComputeImageShare(imageName, []string(shareWith))
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
return
log.Fatalf("[GCP] Sharing image failed: %s", err)
}
}
}

32
cmd/osbuild-worker/jobimpl-osbuild.go
View file

@ -280,19 +280,43 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error {
continue
}
err = g.Upload(path.Join(outputDirectory, options.Filename), options.Bucket, options.Object)
log.Printf("[GCP] 🚀 Uploading image to: %s/%s", options.Bucket, options.Object)
_, err = g.StorageObjectUpload(path.Join(outputDirectory, options.Filename), options.Bucket, options.Object)
if err != nil {
r = append(r, err)
continue
}
err = g.Import(options.Bucket, options.Object, t.ImageName, options.Os, options.Region)
if err != nil {
log.Printf("[GCP] 📥 Importing image into Compute Node as '%s'", t.ImageName)
imageBuild, importErr := g.ComputeImageImport(options.Bucket, options.Object, t.ImageName, options.Os, options.Region)
if imageBuild != nil {
log.Printf("[GCP] 📜 Image import log URL: %s", imageBuild.LogUrl)
log.Printf("[GCP] 🎉 Image import finished with status: %s", imageBuild.Status)
}
// Cleanup storage before checking for errors
log.Printf("[GCP] 🧹 Deleting uploaded image file: %s/%s", options.Bucket, options.Object)
if err = g.StorageObjectDelete(options.Bucket, options.Object); err != nil {
log.Printf("[GCP] Encountered error while deleting object: %v", err)
}
deleted, errs := g.StorageImageImportCleanup(t.ImageName)
for _, d := range deleted {
log.Printf("[GCP] 🧹 Deleted image import job file '%s'", d)
}
for _, e := range errs {
log.Printf("[GCP] Encountered error during image import cleanup: %v", e)
}
// check error from ComputeImageImport()
if importErr != nil {
r = append(r, err)
continue
}
log.Printf("[GCP] 💿 Image URL: %s", g.ComputeImageURL(t.ImageName))
err = g.Share(t.ImageName, options.ShareWithAccounts)
log.Printf("[GCP] 🔗 Sharing the image with: %+v", options.ShareWithAccounts)
err = g.ComputeImageShare(t.ImageName, options.ShareWithAccounts)
if err != nil {
r = append(r, err)
continue