From 01faa858d49c82099814802277eaf98d4ed8b07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Tue, 12 Aug 2025 17:07:20 +0200 Subject: [PATCH] internal/cloud: delete gcp upload implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomáš Hozza --- go.mod | 4 +- internal/cloud/gcp/compute.go | 328 ---------------------------------- internal/cloud/gcp/gcp.go | 70 -------- internal/cloud/gcp/storage.go | 99 ---------- 4 files changed, 2 insertions(+), 499 deletions(-) delete mode 100644 internal/cloud/gcp/compute.go delete mode 100644 internal/cloud/gcp/gcp.go delete mode 100644 internal/cloud/gcp/storage.go diff --git a/go.mod b/go.mod index 70b84b487..73e156e05 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ exclude github.com/mattn/go-sqlite3 v2.0.3+incompatible require ( cloud.google.com/go/compute v1.42.0 - cloud.google.com/go/storage v1.56.0 github.com/BurntSushi/toml v1.5.1-0.20250403130103-3d3abc24416a github.com/aws/aws-sdk-go-v2 v1.37.2 github.com/aws/aws-sdk-go-v2/config v1.30.3 @@ -43,7 +42,6 @@ require ( github.com/spf13/cobra v1.9.1 github.com/stretchr/testify v1.10.0 golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 - golang.org/x/oauth2 v0.30.0 golang.org/x/sync v0.16.0 google.golang.org/api v0.246.0 ) @@ -56,6 +54,7 @@ require ( cloud.google.com/go/compute/metadata v0.7.0 // indirect cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/monitoring v1.24.2 // indirect + cloud.google.com/go/storage v1.56.0 // indirect dario.cat/mergo v1.0.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.11.0 // indirect @@ -223,6 +222,7 @@ require ( golang.org/x/crypto v0.41.0 // indirect golang.org/x/mod v0.27.0 // indirect golang.org/x/net v0.43.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sys v0.35.0 // indirect golang.org/x/term v0.34.0 // indirect golang.org/x/text v0.28.0 // indirect diff --git a/internal/cloud/gcp/compute.go b/internal/cloud/gcp/compute.go deleted file mode 100644 index e7ccc335d..000000000 --- a/internal/cloud/gcp/compute.go +++ /dev/null @@ -1,328 +0,0 @@ -package gcp - -import ( - "context" - "fmt" - "strings" - - compute "cloud.google.com/go/compute/apiv1" - "cloud.google.com/go/compute/apiv1/computepb" - "google.golang.org/api/option" - - "github.com/osbuild/osbuild-composer/internal/common" -) - -// Guest OS Features for RHEL8 images -var GuestOsFeaturesRHEL8 []*computepb.GuestOsFeature = []*computepb.GuestOsFeature{ - {Type: common.ToPtr(computepb.GuestOsFeature_UEFI_COMPATIBLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_VIRTIO_SCSI_MULTIQUEUE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_SEV_CAPABLE.String())}, -} - -// Guest OS Features for RHEL9 images. Note that if you update this, also -// consider changing the code in https://github.com/coreos/coreos-assembler/blob/0083086c4720b602b8243effb85c0a1f73f013dd/mantle/platform/api/gcloud/image.go#L105 -// for RHEL CoreOS which uses coreos-assembler today. -var GuestOsFeaturesRHEL9 []*computepb.GuestOsFeature = []*computepb.GuestOsFeature{ - {Type: common.ToPtr(computepb.GuestOsFeature_UEFI_COMPATIBLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_VIRTIO_SCSI_MULTIQUEUE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_SEV_CAPABLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_GVNIC.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_SEV_SNP_CAPABLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_SEV_LIVE_MIGRATABLE_V2.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_TDX_CAPABLE.String())}, -} - -// Guest OS Features for RHEL images up to RHEL9.5. -// The TDX support was added since RHEL-9.6. -var GuestOsFeaturesRHEL95 []*computepb.GuestOsFeature = []*computepb.GuestOsFeature{ - {Type: common.ToPtr(computepb.GuestOsFeature_UEFI_COMPATIBLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_VIRTIO_SCSI_MULTIQUEUE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_SEV_CAPABLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_GVNIC.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_SEV_SNP_CAPABLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_SEV_LIVE_MIGRATABLE_V2.String())}, -} - -// Guest OS Features for RHEL9.1 images. -// The SEV_LIVE_MIGRATABLE_V2 support was added since RHEL-9.2 -var GuestOsFeaturesRHEL91 []*computepb.GuestOsFeature = []*computepb.GuestOsFeature{ - {Type: common.ToPtr(computepb.GuestOsFeature_UEFI_COMPATIBLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_VIRTIO_SCSI_MULTIQUEUE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_SEV_CAPABLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_GVNIC.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_SEV_SNP_CAPABLE.String())}, -} - -// Guest OS Features for RHEL9.0 images. -// The SEV-SNP support was added since RHEL-9.1, so keeping this for RHEL-9.0 only. -var GuestOsFeaturesRHEL90 []*computepb.GuestOsFeature = []*computepb.GuestOsFeature{ - {Type: common.ToPtr(computepb.GuestOsFeature_UEFI_COMPATIBLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_VIRTIO_SCSI_MULTIQUEUE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_SEV_CAPABLE.String())}, - {Type: common.ToPtr(computepb.GuestOsFeature_GVNIC.String())}, -} - -// Guest OS Features for RHEL-10 images. -var GuestOsFeaturesRHEL10 []*computepb.GuestOsFeature = GuestOsFeaturesRHEL9 - -// GuestOsFeaturesByDistro returns the the list of Guest OS Features, which -// should be used when importing an image of the specified distribution. -// -// In case the provided distribution does not have any specific Guest OS -// Features list defined, nil is returned. -func GuestOsFeaturesByDistro(distroName string) []*computepb.GuestOsFeature { - switch { - case strings.HasPrefix(distroName, "centos-8"): - fallthrough - case strings.HasPrefix(distroName, "rhel-8"): - return GuestOsFeaturesRHEL8 - - // TODO: this should be updated for the dot-notation - case distroName == "rhel-90": - return GuestOsFeaturesRHEL90 - // TODO: this should be updated for the dot-notation - case distroName == "rhel-91": - return GuestOsFeaturesRHEL91 - case distroName == "rhel-92": - fallthrough - case distroName == "rhel-93": - fallthrough - case distroName == "rhel-94": - fallthrough - case distroName == "rhel-95": - return GuestOsFeaturesRHEL95 - case strings.HasPrefix(distroName, "centos-9"): - fallthrough - case strings.HasPrefix(distroName, "rhel-9"): - return GuestOsFeaturesRHEL9 - - case strings.HasPrefix(distroName, "centos-10"): - fallthrough - case strings.HasPrefix(distroName, "rhel-10"): - return GuestOsFeaturesRHEL10 - - default: - return nil - } -} - -// ComputeImageInsert imports a previously uploaded archive with raw image into Compute Engine. -// -// The image must be RAW image named 'disk.raw' inside a gzip-ed tarball. -// -// To delete the Storage object (image) used for the image import, use StorageObjectDelete(). -// -// bucket - Google storage bucket name with the uploaded image archive -// object - Google storage object name of the uploaded image -// imageName - Desired image name after the import. This must be unique within the whole project. -// regions - A list of valid Google Storage regions where the resulting image should be located. -// -// It is possible to specify multiple regions. Also multi and dual regions are allowed. -// If not provided, the region of the used Storage object is used. -// See: https://cloud.google.com/storage/docs/locations -// -// guestOsFeatures - A list of features supported by the Guest OS on the imported image. -// -// Uses: -// - Compute Engine API -func (g *GCP) ComputeImageInsert( - ctx context.Context, - bucket, object, imageName string, - regions []string, - guestOsFeatures []*computepb.GuestOsFeature) (*computepb.Image, error) { - imagesClient, err := compute.NewImagesRESTClient(ctx, option.WithCredentials(g.creds)) - if err != nil { - return nil, fmt.Errorf("failed to get Compute Engine Images client: %v", err) - } - defer imagesClient.Close() - - operationsClient, err := compute.NewGlobalOperationsRESTClient(ctx, option.WithCredentials(g.creds)) - if err != nil { - return nil, fmt.Errorf("failed to get Compute Engine Operations client: %v", err) - } - defer operationsClient.Close() - - imgInsertReq := &computepb.InsertImageRequest{ - Project: g.GetProjectID(), - ImageResource: &computepb.Image{ - Name: &imageName, - StorageLocations: regions, - GuestOsFeatures: guestOsFeatures, - RawDisk: &computepb.RawDisk{ - ContainerType: common.ToPtr(computepb.RawDisk_TAR.String()), - Source: common.ToPtr(fmt.Sprintf("https://storage.googleapis.com/%s/%s", bucket, object)), - }, - }, - } - - operation, err := imagesClient.Insert(ctx, imgInsertReq) - if err != nil { - return nil, fmt.Errorf("failed to insert provided image into GCE: %v", err) - } - - // wait for the operation to finish - var operationResource *computepb.Operation - for { - waitOperationReq := &computepb.WaitGlobalOperationRequest{ - Operation: operation.Proto().GetName(), - Project: g.GetProjectID(), - } - - operationResource, err = operationsClient.Wait(ctx, waitOperationReq) - if err != nil { - return nil, fmt.Errorf("failed to wait for an Image Import operation: %v", err) - } - - // The operation finished - if operationResource.GetStatus() != computepb.Operation_RUNNING && operationResource.GetStatus() != computepb.Operation_PENDING { - break - } - } - - // If the operation failed, the HttpErrorStatusCode is set to a non-zero value - if operationStatusCode := operationResource.GetHttpErrorStatusCode(); operationStatusCode != 0 { - operationErrorMsg := operationResource.GetHttpErrorMessage() - operationErrors := operationResource.GetError().GetErrors() - return nil, fmt.Errorf("failed to insert image into GCE. HTTPErrorCode:%d HTTPErrorMsg:%v Errors:%v", operationStatusCode, operationErrorMsg, operationErrors) - } - - getImageReq := &computepb.GetImageRequest{ - Image: imageName, - Project: g.GetProjectID(), - } - - image, err := imagesClient.Get(ctx, getImageReq) - if err != nil { - return nil, fmt.Errorf("failed to get information about the imported Image: %v", err) - } - - return image, nil -} - -// ComputeImageURL returns an image's URL to Google Cloud Console. The method does -// not check at all, if the image actually exists or not. -func (g *GCP) ComputeImageURL(imageName string) string { - return fmt.Sprintf("https://console.cloud.google.com/compute/imagesDetail/projects/%s/global/images/%s", g.GetProjectID(), imageName) -} - -// ComputeImageShare shares the specified Compute Engine image with list of accounts. -// -// "shareWith" is a list of accounts to share the image with. Items can be one -// of the following options: -// -// - `user:{emailid}`: An email address that represents a specific -// Google account. For example, `alice@example.com`. -// -// - `serviceAccount:{emailid}`: An email address that represents a -// service account. For example, `my-other-app@appspot.gserviceaccount.com`. -// -// - `group:{emailid}`: An email address that represents a Google group. -// For example, `admins@example.com`. -// -// - `domain:{domain}`: The G Suite domain (primary) that represents all -// the users of that domain. For example, `google.com` or `example.com`. -// -// Uses: -// - Compute Engine API -func (g *GCP) ComputeImageShare(ctx context.Context, imageName string, shareWith []string) error { - imagesClient, err := compute.NewImagesRESTClient(ctx, option.WithCredentials(g.creds)) - if err != nil { - return fmt.Errorf("failed to get Compute Engine Images client: %v", err) - } - defer imagesClient.Close() - - // Standard role to enable account to view and use a specific Image - imageDesiredRole := "roles/compute.imageUser" - - // Get the current Policy set on the Image - getIamPolicyReq := &computepb.GetIamPolicyImageRequest{ - Project: g.GetProjectID(), - Resource: imageName, - } - policy, err := imagesClient.GetIamPolicy(ctx, getIamPolicyReq) - if err != nil { - return fmt.Errorf("failed to get image's policy: %v", err) - } - - // Add new members, who can use the image - // Completely override the old policy - userBinding := &computepb.Binding{ - Members: shareWith, - Role: common.ToPtr(imageDesiredRole), - } - newPolicy := &computepb.Policy{ - Bindings: []*computepb.Binding{userBinding}, - Etag: policy.Etag, - } - setIamPolicyReq := &computepb.SetIamPolicyImageRequest{ - Project: g.GetProjectID(), - Resource: imageName, - GlobalSetPolicyRequestResource: &computepb.GlobalSetPolicyRequest{ - Policy: newPolicy, - }, - } - _, err = imagesClient.SetIamPolicy(ctx, setIamPolicyReq) - if err != nil { - return fmt.Errorf("failed to set new image policy: %v", err) - } - - // Users won't see the shared image in their images.list requests, unless - // they are also granted a specific "imagesList" role on the project. If you - // don't need users to be able to view the list of shared images, this - // step can be skipped. - // - // Downside of granting the "imagesList" role to a project is that the user - // will be able to list all available images in the project, even those that - // they can't use because of insufficient permissions. - // - // Even without the ability to view / list shared images, the user can still - // create a Compute Engine instance using the image via API or 'gcloud' tool. - // - // Custom role to enable account to only list images in the project. - // Without this role, the account won't be able to list and see the image - // in the GCP Web UI. - - // For now, the decision is that the account should not get any role to the - // project, where the image has been imported. - - return nil -} - -// ComputeImageDelete deletes a Compute Engine image with the given name. If the -// image existed and was successfully deleted, no error is returned. -// -// Uses: -// - Compute Engine API -func (g *GCP) ComputeImageDelete(ctx context.Context, name string) error { - imagesClient, err := compute.NewImagesRESTClient(ctx, option.WithCredentials(g.creds)) - if err != nil { - return fmt.Errorf("failed to get Compute Engine Images client: %v", err) - } - defer imagesClient.Close() - - req := &computepb.DeleteImageRequest{ - Project: g.GetProjectID(), - Image: name, - } - _, err = imagesClient.Delete(ctx, req) - - return err -} - -// ComputeExecuteFunctionForImages will pass all the compute images in the account to a function, -// which is able to iterate over the images. Useful if something needs to be execute for each image. -// Uses: -// - Compute Engine API -func (g *GCP) ComputeExecuteFunctionForImages(ctx context.Context, f func(*compute.ImageIterator) error) error { - imagesClient, err := compute.NewImagesRESTClient(ctx, option.WithCredentials(g.creds)) - if err != nil { - return fmt.Errorf("failed to get Compute Engine Images client: %v", err) - } - defer imagesClient.Close() - - req := &computepb.ListImagesRequest{ - Project: g.GetProjectID(), - } - imagesIterator := imagesClient.List(ctx, req) - return f(imagesIterator) -} diff --git a/internal/cloud/gcp/gcp.go b/internal/cloud/gcp/gcp.go deleted file mode 100644 index 6b5948363..000000000 --- a/internal/cloud/gcp/gcp.go +++ /dev/null @@ -1,70 +0,0 @@ -package gcp - -import ( - "context" - "fmt" - "os" - - compute "cloud.google.com/go/compute/apiv1" - "cloud.google.com/go/storage" - "golang.org/x/oauth2/google" -) - -// GCPCredentialsEnvName contains name of the environment variable used -// to specify the path to file with CGP service account credentials -const ( - //nolint:gosec - GCPCredentialsEnvName string = "GOOGLE_APPLICATION_CREDENTIALS" -) - -// GCP structure holds necessary information to authenticate and interact with GCP. -type GCP struct { - creds *google.Credentials -} - -// New returns an authenticated GCP instance, allowing to interact with GCP API. -func New(credentials []byte) (*GCP, error) { - scopes := []string{storage.ScopeReadWrite} // file upload - scopes = append(scopes, compute.DefaultAuthScopes()...) // permissions to image - - var getCredsFunc func() (*google.Credentials, error) - if credentials != nil { - getCredsFunc = func() (*google.Credentials, error) { - return google.CredentialsFromJSON( - context.Background(), - credentials, - scopes..., - ) - } - } else { - getCredsFunc = func() (*google.Credentials, error) { - return google.FindDefaultCredentials( - context.Background(), - scopes..., - ) - } - } - - creds, err := getCredsFunc() - if err != nil { - return nil, fmt.Errorf("failed to get Google credentials: %v", err) - } - - return &GCP{creds}, nil -} - -// NewFromFile loads the credentials from a file and returns an authenticated -// *GCP object instance. -func NewFromFile(path string) (*GCP, error) { - gcpCredentials, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("cannot load GCP credentials from file %q: %v", path, err) - } - return New(gcpCredentials) -} - -// GetProjectID returns a string with the Project ID of the project, used for -// all GCP operations. -func (g *GCP) GetProjectID() string { - return g.creds.ProjectID -} diff --git a/internal/cloud/gcp/storage.go b/internal/cloud/gcp/storage.go deleted file mode 100644 index 8dadd29c7..000000000 --- a/internal/cloud/gcp/storage.go +++ /dev/null @@ -1,99 +0,0 @@ -package gcp - -import ( - "context" - // gcp uses MD5 hashes - /* #nosec G501 */ - "crypto/md5" - "fmt" - "io" - "os" - - "cloud.google.com/go/storage" - "google.golang.org/api/option" -) - -const ( - // MetadataKeyImageName contains a key name used to store metadata on - // a Storage object with the intended name of the image. - // The metadata can be then used to associate the object with actual - // image build using the image name. - MetadataKeyImageName string = "osbuild-composer-image-name" -) - -// StorageObjectUpload uploads an OS image to specified Cloud Storage bucket and object. -// The bucket must exist. MD5 sum of the image file and uploaded object is -// compared after the upload to verify the integrity of the uploaded image. -// -// The ObjectAttrs is returned if the object has been created. -// -// Uses: -// - Storage API -func (g *GCP) StorageObjectUpload(ctx context.Context, filename, bucket, object string, metadata map[string]string) (*storage.ObjectAttrs, error) { - storageClient, err := storage.NewClient(ctx, option.WithCredentials(g.creds)) - if err != nil { - return nil, fmt.Errorf("failed to get Storage client: %v", err) - } - defer storageClient.Close() - - // Open the image file - imageFile, err := os.Open(filename) - if err != nil { - return nil, fmt.Errorf("cannot open the image: %v", err) - } - defer imageFile.Close() - - // Compute MD5 checksum of the image file for later verification - // gcp uses MD5 hashes - /* #nosec G401 */ - imageFileHash := md5.New() - if _, err := io.Copy(imageFileHash, imageFile); err != nil { - return nil, fmt.Errorf("cannot create md5 of the image: %v", err) - } - // Move the cursor of opened file back to the start - if _, err := imageFile.Seek(0, 0); err != nil { - return nil, fmt.Errorf("cannot seek the image: %v", err) - } - - // Upload the image - // The Bucket MUST exist and be of a STANDARD storage class - obj := storageClient.Bucket(bucket).Object(object) - wc := obj.NewWriter(ctx) - - // Uploaded data is rejected if its MD5 hash does not match the set value. - wc.MD5 = imageFileHash.Sum(nil) - - if metadata != nil { - wc.ObjectAttrs.Metadata = metadata - } - - if _, err = io.Copy(wc, imageFile); err != nil { - return nil, fmt.Errorf("uploading the image failed: %v", err) - } - - // The object will not be available until Close has been called. - if err := wc.Close(); err != nil { - return nil, fmt.Errorf("Writer.Close: %v", err) - } - - return wc.Attrs(), nil -} - -// StorageObjectDelete deletes the given object from a bucket. -// -// Uses: -// - Storage API -func (g *GCP) StorageObjectDelete(ctx context.Context, bucket, object string) error { - storageClient, err := storage.NewClient(ctx, option.WithCredentials(g.creds)) - if err != nil { - return fmt.Errorf("failed to get Storage client: %v", err) - } - defer storageClient.Close() - - objectHandle := storageClient.Bucket(bucket).Object(object) - if err = objectHandle.Delete(ctx); err != nil { - return fmt.Errorf("failed to delete image file object: %v", err) - } - - return nil -}