particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

internal/cloud: delete gcp upload implementation

Browse source 
Signed-off-by: Tomáš Hozza <thozza@redhat.com>
This commit is contained in:
Tomáš Hozza 2025-08-12 17:07:20 +02:00 • committed by Achilleas Koutsou
parent 992c6365f0
commit 01faa858d4
4 changed files with 2 additions and 499 deletions
Download patch file Download diff file Expand all files Collapse all files

4
go.mod
View file

@ -6,7 +6,6 @@ exclude github.com/mattn/go-sqlite3 v2.0.3+incompatible
require ( require (
cloud.google.com/go/compute v1.42.0 cloud.google.com/go/compute v1.42.0
cloud.google.com/go/storage v1.56.0
github.com/BurntSushi/toml v1.5.1-0.20250403130103-3d3abc24416a github.com/BurntSushi/toml v1.5.1-0.20250403130103-3d3abc24416a
github.com/aws/aws-sdk-go-v2 v1.37.2 github.com/aws/aws-sdk-go-v2 v1.37.2
github.com/aws/aws-sdk-go-v2/config v1.30.3 github.com/aws/aws-sdk-go-v2/config v1.30.3
@ -43,7 +42,6 @@ require (
github.com/spf13/cobra v1.9.1 github.com/spf13/cobra v1.9.1
github.com/stretchr/testify v1.10.0 github.com/stretchr/testify v1.10.0
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329 golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
golang.org/x/oauth2 v0.30.0
golang.org/x/sync v0.16.0 golang.org/x/sync v0.16.0
google.golang.org/api v0.246.0 google.golang.org/api v0.246.0
) )
@ -56,6 +54,7 @@ require (
cloud.google.com/go/compute/metadata v0.7.0 // indirect cloud.google.com/go/compute/metadata v0.7.0 // indirect
cloud.google.com/go/iam v1.5.2 // indirect cloud.google.com/go/iam v1.5.2 // indirect
cloud.google.com/go/monitoring v1.24.2 // indirect cloud.google.com/go/monitoring v1.24.2 // indirect
cloud.google.com/go/storage v1.56.0 // indirect
dario.cat/mergo v1.0.2 // indirect dario.cat/mergo v1.0.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.11.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.11.0 // indirect
@ -223,6 +222,7 @@ require (
golang.org/x/crypto v0.41.0 // indirect golang.org/x/crypto v0.41.0 // indirect
golang.org/x/mod v0.27.0 // indirect golang.org/x/mod v0.27.0 // indirect
golang.org/x/net v0.43.0 // indirect golang.org/x/net v0.43.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sys v0.35.0 // indirect golang.org/x/sys v0.35.0 // indirect
golang.org/x/term v0.34.0 // indirect golang.org/x/term v0.34.0 // indirect
golang.org/x/text v0.28.0 // indirect golang.org/x/text v0.28.0 // indirect

328
internal/cloud/gcp/compute.go
View file

@ -1,328 +0,0 @@
package gcp
import (
"context"
"fmt"
"strings"
compute "cloud.google.com/go/compute/apiv1"
"cloud.google.com/go/compute/apiv1/computepb"
"google.golang.org/api/option"
"github.com/osbuild/osbuild-composer/internal/common"
)
// Guest OS Features for RHEL8 images
var GuestOsFeaturesRHEL8 []*computepb.GuestOsFeature = []*computepb.GuestOsFeature{
{Type: common.ToPtr(computepb.GuestOsFeature_UEFI_COMPATIBLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_VIRTIO_SCSI_MULTIQUEUE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_SEV_CAPABLE.String())},
}
// Guest OS Features for RHEL9 images. Note that if you update this, also
// consider changing the code in https://github.com/coreos/coreos-assembler/blob/0083086c4720b602b8243effb85c0a1f73f013dd/mantle/platform/api/gcloud/image.go#L105
// for RHEL CoreOS which uses coreos-assembler today.
var GuestOsFeaturesRHEL9 []*computepb.GuestOsFeature = []*computepb.GuestOsFeature{
{Type: common.ToPtr(computepb.GuestOsFeature_UEFI_COMPATIBLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_VIRTIO_SCSI_MULTIQUEUE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_SEV_CAPABLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_GVNIC.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_SEV_SNP_CAPABLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_SEV_LIVE_MIGRATABLE_V2.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_TDX_CAPABLE.String())},
}
// Guest OS Features for RHEL images up to RHEL9.5.
// The TDX support was added since RHEL-9.6.
var GuestOsFeaturesRHEL95 []*computepb.GuestOsFeature = []*computepb.GuestOsFeature{
{Type: common.ToPtr(computepb.GuestOsFeature_UEFI_COMPATIBLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_VIRTIO_SCSI_MULTIQUEUE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_SEV_CAPABLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_GVNIC.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_SEV_SNP_CAPABLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_SEV_LIVE_MIGRATABLE_V2.String())},
}
// Guest OS Features for RHEL9.1 images.
// The SEV_LIVE_MIGRATABLE_V2 support was added since RHEL-9.2
var GuestOsFeaturesRHEL91 []*computepb.GuestOsFeature = []*computepb.GuestOsFeature{
{Type: common.ToPtr(computepb.GuestOsFeature_UEFI_COMPATIBLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_VIRTIO_SCSI_MULTIQUEUE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_SEV_CAPABLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_GVNIC.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_SEV_SNP_CAPABLE.String())},
}
// Guest OS Features for RHEL9.0 images.
// The SEV-SNP support was added since RHEL-9.1, so keeping this for RHEL-9.0 only.
var GuestOsFeaturesRHEL90 []*computepb.GuestOsFeature = []*computepb.GuestOsFeature{
{Type: common.ToPtr(computepb.GuestOsFeature_UEFI_COMPATIBLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_VIRTIO_SCSI_MULTIQUEUE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_SEV_CAPABLE.String())},
{Type: common.ToPtr(computepb.GuestOsFeature_GVNIC.String())},
}
// Guest OS Features for RHEL-10 images.
var GuestOsFeaturesRHEL10 []*computepb.GuestOsFeature = GuestOsFeaturesRHEL9
// GuestOsFeaturesByDistro returns the the list of Guest OS Features, which
// should be used when importing an image of the specified distribution.
//
// In case the provided distribution does not have any specific Guest OS
// Features list defined, nil is returned.
func GuestOsFeaturesByDistro(distroName string) []*computepb.GuestOsFeature {
switch {
case strings.HasPrefix(distroName, "centos-8"):
fallthrough
case strings.HasPrefix(distroName, "rhel-8"):
return GuestOsFeaturesRHEL8
// TODO: this should be updated for the dot-notation
case distroName == "rhel-90":
return GuestOsFeaturesRHEL90
// TODO: this should be updated for the dot-notation
case distroName == "rhel-91":
return GuestOsFeaturesRHEL91
case distroName == "rhel-92":
fallthrough
case distroName == "rhel-93":
fallthrough
case distroName == "rhel-94":
fallthrough
case distroName == "rhel-95":
return GuestOsFeaturesRHEL95
case strings.HasPrefix(distroName, "centos-9"):
fallthrough
case strings.HasPrefix(distroName, "rhel-9"):
return GuestOsFeaturesRHEL9
case strings.HasPrefix(distroName, "centos-10"):
fallthrough
case strings.HasPrefix(distroName, "rhel-10"):
return GuestOsFeaturesRHEL10
default:
return nil
}
}
// ComputeImageInsert imports a previously uploaded archive with raw image into Compute Engine.
//
// The image must be RAW image named 'disk.raw' inside a gzip-ed tarball.
//
// To delete the Storage object (image) used for the image import, use StorageObjectDelete().
//
// bucket - Google storage bucket name with the uploaded image archive
// object - Google storage object name of the uploaded image
// imageName - Desired image name after the import. This must be unique within the whole project.
// regions - A list of valid Google Storage regions where the resulting image should be located.
//
// It is possible to specify multiple regions. Also multi and dual regions are allowed.
// If not provided, the region of the used Storage object is used.
// See: https://cloud.google.com/storage/docs/locations
//
// guestOsFeatures - A list of features supported by the Guest OS on the imported image.
//
// Uses:
// - Compute Engine API
func (g *GCP) ComputeImageInsert(
ctx context.Context,
bucket, object, imageName string,
regions []string,
guestOsFeatures []*computepb.GuestOsFeature) (*computepb.Image, error) {
imagesClient, err := compute.NewImagesRESTClient(ctx, option.WithCredentials(g.creds))
if err != nil {
return nil, fmt.Errorf("failed to get Compute Engine Images client: %v", err)
}
defer imagesClient.Close()
operationsClient, err := compute.NewGlobalOperationsRESTClient(ctx, option.WithCredentials(g.creds))
if err != nil {
return nil, fmt.Errorf("failed to get Compute Engine Operations client: %v", err)
}
defer operationsClient.Close()
imgInsertReq := &computepb.InsertImageRequest{
Project: g.GetProjectID(),
ImageResource: &computepb.Image{
Name: &imageName,
StorageLocations: regions,
GuestOsFeatures: guestOsFeatures,
RawDisk: &computepb.RawDisk{
ContainerType: common.ToPtr(computepb.RawDisk_TAR.String()),
Source: common.ToPtr(fmt.Sprintf("https://storage.googleapis.com/%s/%s", bucket, object)),
},
},
}
operation, err := imagesClient.Insert(ctx, imgInsertReq)
if err != nil {
return nil, fmt.Errorf("failed to insert provided image into GCE: %v", err)
}
// wait for the operation to finish
var operationResource *computepb.Operation
for {
waitOperationReq := &computepb.WaitGlobalOperationRequest{
Operation: operation.Proto().GetName(),
Project: g.GetProjectID(),
}
operationResource, err = operationsClient.Wait(ctx, waitOperationReq)
if err != nil {
return nil, fmt.Errorf("failed to wait for an Image Import operation: %v", err)
}
// The operation finished
if operationResource.GetStatus() != computepb.Operation_RUNNING && operationResource.GetStatus() != computepb.Operation_PENDING {
break
}
}
// If the operation failed, the HttpErrorStatusCode is set to a non-zero value
if operationStatusCode := operationResource.GetHttpErrorStatusCode(); operationStatusCode != 0 {
operationErrorMsg := operationResource.GetHttpErrorMessage()
operationErrors := operationResource.GetError().GetErrors()
return nil, fmt.Errorf("failed to insert image into GCE. HTTPErrorCode:%d HTTPErrorMsg:%v Errors:%v", operationStatusCode, operationErrorMsg, operationErrors)
}
getImageReq := &computepb.GetImageRequest{
Image: imageName,
Project: g.GetProjectID(),
}
image, err := imagesClient.Get(ctx, getImageReq)
if err != nil {
return nil, fmt.Errorf("failed to get information about the imported Image: %v", err)
}
return image, nil
}
// ComputeImageURL returns an image's URL to Google Cloud Console. The method does
// not check at all, if the image actually exists or not.
func (g *GCP) ComputeImageURL(imageName string) string {
return fmt.Sprintf("https://console.cloud.google.com/compute/imagesDetail/projects/%s/global/images/%s", g.GetProjectID(), imageName)
}
// ComputeImageShare shares the specified Compute Engine image with list of accounts.
//
// "shareWith" is a list of accounts to share the image with. Items can be one
// of the following options:
//
// - `user:{emailid}`: An email address that represents a specific
// Google account. For example, `alice@example.com`.
//
// - `serviceAccount:{emailid}`: An email address that represents a
// service account. For example, `my-other-app@appspot.gserviceaccount.com`.
//
// - `group:{emailid}`: An email address that represents a Google group.
// For example, `admins@example.com`.
//
// - `domain:{domain}`: The G Suite domain (primary) that represents all
// the users of that domain. For example, `google.com` or `example.com`.
//
// Uses:
// - Compute Engine API
func (g *GCP) ComputeImageShare(ctx context.Context, imageName string, shareWith []string) error {
imagesClient, err := compute.NewImagesRESTClient(ctx, option.WithCredentials(g.creds))
if err != nil {
return fmt.Errorf("failed to get Compute Engine Images client: %v", err)
}
defer imagesClient.Close()
// Standard role to enable account to view and use a specific Image
imageDesiredRole := "roles/compute.imageUser"
// Get the current Policy set on the Image
getIamPolicyReq := &computepb.GetIamPolicyImageRequest{
Project: g.GetProjectID(),
Resource: imageName,
}
policy, err := imagesClient.GetIamPolicy(ctx, getIamPolicyReq)
if err != nil {
return fmt.Errorf("failed to get image's policy: %v", err)
}
// Add new members, who can use the image
// Completely override the old policy
userBinding := &computepb.Binding{
Members: shareWith,
Role: common.ToPtr(imageDesiredRole),
}
newPolicy := &computepb.Policy{
Bindings: []*computepb.Binding{userBinding},
Etag: policy.Etag,
}
setIamPolicyReq := &computepb.SetIamPolicyImageRequest{
Project: g.GetProjectID(),
Resource: imageName,
GlobalSetPolicyRequestResource: &computepb.GlobalSetPolicyRequest{
Policy: newPolicy,
},
}
_, err = imagesClient.SetIamPolicy(ctx, setIamPolicyReq)
if err != nil {
return fmt.Errorf("failed to set new image policy: %v", err)
}
// Users won't see the shared image in their images.list requests, unless
// they are also granted a specific "imagesList" role on the project. If you
// don't need users to be able to view the list of shared images, this
// step can be skipped.
//
// Downside of granting the "imagesList" role to a project is that the user
// will be able to list all available images in the project, even those that
// they can't use because of insufficient permissions.
//
// Even without the ability to view / list shared images, the user can still
// create a Compute Engine instance using the image via API or 'gcloud' tool.
//
// Custom role to enable account to only list images in the project.
// Without this role, the account won't be able to list and see the image
// in the GCP Web UI.
// For now, the decision is that the account should not get any role to the
// project, where the image has been imported.
return nil
}
// ComputeImageDelete deletes a Compute Engine image with the given name. If the
// image existed and was successfully deleted, no error is returned.
//
// Uses:
// - Compute Engine API
func (g *GCP) ComputeImageDelete(ctx context.Context, name string) error {
imagesClient, err := compute.NewImagesRESTClient(ctx, option.WithCredentials(g.creds))
if err != nil {
return fmt.Errorf("failed to get Compute Engine Images client: %v", err)
}
defer imagesClient.Close()
req := &computepb.DeleteImageRequest{
Project: g.GetProjectID(),
Image: name,
}
_, err = imagesClient.Delete(ctx, req)
return err
}
// ComputeExecuteFunctionForImages will pass all the compute images in the account to a function,
// which is able to iterate over the images. Useful if something needs to be execute for each image.
// Uses:
// - Compute Engine API
func (g *GCP) ComputeExecuteFunctionForImages(ctx context.Context, f func(*compute.ImageIterator) error) error {
imagesClient, err := compute.NewImagesRESTClient(ctx, option.WithCredentials(g.creds))
if err != nil {
return fmt.Errorf("failed to get Compute Engine Images client: %v", err)
}
defer imagesClient.Close()
req := &computepb.ListImagesRequest{
Project: g.GetProjectID(),
}
imagesIterator := imagesClient.List(ctx, req)
return f(imagesIterator)
}

70
internal/cloud/gcp/gcp.go
View file

@ -1,70 +0,0 @@
package gcp
import (
"context"
"fmt"
"os"
compute "cloud.google.com/go/compute/apiv1"
"cloud.google.com/go/storage"
"golang.org/x/oauth2/google"
)
// GCPCredentialsEnvName contains name of the environment variable used
// to specify the path to file with CGP service account credentials
const (
//nolint:gosec
GCPCredentialsEnvName string = "GOOGLE_APPLICATION_CREDENTIALS"
)
// GCP structure holds necessary information to authenticate and interact with GCP.
type GCP struct {
creds *google.Credentials
}
// New returns an authenticated GCP instance, allowing to interact with GCP API.
func New(credentials []byte) (*GCP, error) {
scopes := []string{storage.ScopeReadWrite} // file upload
scopes = append(scopes, compute.DefaultAuthScopes()...) // permissions to image
var getCredsFunc func() (*google.Credentials, error)
if credentials != nil {
getCredsFunc = func() (*google.Credentials, error) {
return google.CredentialsFromJSON(
context.Background(),
credentials,
scopes...,
)
}
} else {
getCredsFunc = func() (*google.Credentials, error) {
return google.FindDefaultCredentials(
context.Background(),
scopes...,
)
}
}
creds, err := getCredsFunc()
if err != nil {
return nil, fmt.Errorf("failed to get Google credentials: %v", err)
}
return &GCP{creds}, nil
}
// NewFromFile loads the credentials from a file and returns an authenticated
// *GCP object instance.
func NewFromFile(path string) (*GCP, error) {
gcpCredentials, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("cannot load GCP credentials from file %q: %v", path, err)
}
return New(gcpCredentials)
}
// GetProjectID returns a string with the Project ID of the project, used for
// all GCP operations.
func (g *GCP) GetProjectID() string {
return g.creds.ProjectID
}

99
internal/cloud/gcp/storage.go
View file

@ -1,99 +0,0 @@
package gcp
import (
"context"
// gcp uses MD5 hashes
/* #nosec G501 */
"crypto/md5"
"fmt"
"io"
"os"
"cloud.google.com/go/storage"
"google.golang.org/api/option"
)
const (
// MetadataKeyImageName contains a key name used to store metadata on
// a Storage object with the intended name of the image.
// The metadata can be then used to associate the object with actual
// image build using the image name.
MetadataKeyImageName string = "osbuild-composer-image-name"
)
// StorageObjectUpload uploads an OS image to specified Cloud Storage bucket and object.
// The bucket must exist. MD5 sum of the image file and uploaded object is
// compared after the upload to verify the integrity of the uploaded image.
//
// The ObjectAttrs is returned if the object has been created.
//
// Uses:
// - Storage API
func (g *GCP) StorageObjectUpload(ctx context.Context, filename, bucket, object string, metadata map[string]string) (*storage.ObjectAttrs, error) {
storageClient, err := storage.NewClient(ctx, option.WithCredentials(g.creds))
if err != nil {
return nil, fmt.Errorf("failed to get Storage client: %v", err)
}
defer storageClient.Close()
// Open the image file
imageFile, err := os.Open(filename)
if err != nil {
return nil, fmt.Errorf("cannot open the image: %v", err)
}
defer imageFile.Close()
// Compute MD5 checksum of the image file for later verification
// gcp uses MD5 hashes
/* #nosec G401 */
imageFileHash := md5.New()
if _, err := io.Copy(imageFileHash, imageFile); err != nil {
return nil, fmt.Errorf("cannot create md5 of the image: %v", err)
}
// Move the cursor of opened file back to the start
if _, err := imageFile.Seek(0, 0); err != nil {
return nil, fmt.Errorf("cannot seek the image: %v", err)
}
// Upload the image
// The Bucket MUST exist and be of a STANDARD storage class
obj := storageClient.Bucket(bucket).Object(object)
wc := obj.NewWriter(ctx)
// Uploaded data is rejected if its MD5 hash does not match the set value.
wc.MD5 = imageFileHash.Sum(nil)
if metadata != nil {
wc.ObjectAttrs.Metadata = metadata
}
if _, err = io.Copy(wc, imageFile); err != nil {
return nil, fmt.Errorf("uploading the image failed: %v", err)
}
// The object will not be available until Close has been called.
if err := wc.Close(); err != nil {
return nil, fmt.Errorf("Writer.Close: %v", err)
}
return wc.Attrs(), nil
}
// StorageObjectDelete deletes the given object from a bucket.
//
// Uses:
// - Storage API
func (g *GCP) StorageObjectDelete(ctx context.Context, bucket, object string) error {
storageClient, err := storage.NewClient(ctx, option.WithCredentials(g.creds))
if err != nil {
return fmt.Errorf("failed to get Storage client: %v", err)
}
defer storageClient.Close()
objectHandle := storageClient.Bucket(bucket).Object(object)
if err = objectHandle.Delete(ctx); err != nil {
return fmt.Errorf("failed to delete image file object: %v", err)
}
return nil
}