diff --git a/cmd/osbuild-upload-azure/main.go b/cmd/osbuild-upload-azure/main.go index 2c25adf62..cfce5e6ff 100644 --- a/cmd/osbuild-upload-azure/main.go +++ b/cmd/osbuild-upload-azure/main.go @@ -8,7 +8,7 @@ import ( "path" "strings" - "github.com/osbuild/osbuild-composer/internal/upload/azure" + "github.com/osbuild/images/pkg/upload/azure" ) func checkStringNotEmpty(variable string, errorMessage string) { diff --git a/cmd/osbuild-worker/config.go b/cmd/osbuild-worker/config.go index fc4ce8f43..0276d573b 100644 --- a/cmd/osbuild-worker/config.go +++ b/cmd/osbuild-worker/config.go @@ -6,7 +6,7 @@ import ( "time" "github.com/BurntSushi/toml" - "github.com/osbuild/osbuild-composer/internal/upload/azure" + "github.com/osbuild/images/pkg/upload/azure" "github.com/sirupsen/logrus" ) diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index c6a652476..2fc269597 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -29,12 +29,12 @@ import ( "github.com/google/uuid" "github.com/sirupsen/logrus" + "github.com/osbuild/images/pkg/upload/azure" "github.com/osbuild/images/pkg/upload/koji" "github.com/osbuild/osbuild-composer/internal/cloud/awscloud" "github.com/osbuild/osbuild-composer/internal/cloud/gcp" "github.com/osbuild/osbuild-composer/internal/osbuildexecutor" "github.com/osbuild/osbuild-composer/internal/target" - "github.com/osbuild/osbuild-composer/internal/upload/azure" "github.com/osbuild/osbuild-composer/internal/upload/vmware" "github.com/osbuild/osbuild-composer/internal/worker" "github.com/osbuild/osbuild-composer/internal/worker/clienterrors" @@ -978,16 +978,19 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { } logWithId.Info("[Azure] 📝 Registering the image") + hyperVGen := azure.HyperVGenV1 + if targetOptions.HyperVGeneration == target.HyperVGenV2 { + hyperVGen = azure.HyperVGenV2 + } err = c.RegisterImage( ctx, - targetOptions.SubscriptionID, targetOptions.ResourceGroup, storageAccount, storageContainer, blobName, jobTarget.ImageName, location, - targetOptions.HyperVGeneration, + hyperVGen, ) if err != nil { targetResult.TargetError = clienterrors.New(clienterrors.ErrorImportingImage, fmt.Sprintf("registering the image failed: %v", err), nil) diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go index 33572b1a6..1cb7c09e0 100644 --- a/cmd/osbuild-worker/main.go +++ b/cmd/osbuild-worker/main.go @@ -21,9 +21,9 @@ import ( "github.com/osbuild/images/pkg/arch" "github.com/osbuild/images/pkg/dnfjson" + "github.com/osbuild/images/pkg/upload/azure" "github.com/osbuild/images/pkg/upload/koji" "github.com/osbuild/osbuild-composer/internal/cloud/awscloud" - "github.com/osbuild/osbuild-composer/internal/upload/azure" "github.com/osbuild/osbuild-composer/internal/upload/oci" "github.com/osbuild/osbuild-composer/internal/worker" ) diff --git a/go.mod b/go.mod index f031fecff..93480cee7 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,6 @@ require ( cloud.google.com/go/compute v1.40.0 cloud.google.com/go/storage v1.55.0 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 github.com/Azure/go-autorest/autorest v0.11.30 github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 @@ -73,7 +69,11 @@ require ( cloud.google.com/go/monitoring v1.24.2 // indirect dario.cat/mergo v1.0.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect diff --git a/internal/boot/azuretest/azure.go b/internal/boot/azuretest/azure.go index 1180e75b4..73113a0db 100644 --- a/internal/boot/azuretest/azure.go +++ b/internal/boot/azuretest/azure.go @@ -21,7 +21,7 @@ import ( "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/osbuild/osbuild-composer/internal/common" - "github.com/osbuild/osbuild-composer/internal/upload/azure" + "github.com/osbuild/images/pkg/upload/azure" ) // wrapErrorf returns error constructed using fmt.Errorf from format and any diff --git a/internal/upload/azure/azurestorage_test.go b/internal/upload/azure/azurestorage_test.go deleted file mode 100644 index 249b4f4a4..000000000 --- a/internal/upload/azure/azurestorage_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package azure - -import ( - "regexp" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRandomStorageAccountName(t *testing.T) { - randomName := RandomStorageAccountName("ib") - - assert.Len(t, randomName, 24) - - r := regexp.MustCompile(`^[\d\w]{24}$`) - assert.True(t, r.MatchString(randomName), "the returned name should be 24 characters long and contain only alphanumerical characters") -} - -func TestEnsureVHDExtension(t *testing.T) { - tests := []struct { - s string - want string - }{ - {s: "toucan.zip", want: "toucan.zip.vhd"}, - {s: "kingfisher.vhd", want: "kingfisher.vhd"}, - } - for _, tt := range tests { - t.Run(tt.s, func(t *testing.T) { - require.Equal(t, tt.want, EnsureVHDExtension(tt.s)) - }) - } -} diff --git a/internal/upload/azure/azure.go b/vendor/github.com/osbuild/images/pkg/upload/azure/azure.go similarity index 84% rename from internal/upload/azure/azure.go rename to vendor/github.com/osbuild/images/pkg/upload/azure/azure.go index b32132814..cb9db45de 100644 --- a/internal/upload/azure/azure.go +++ b/vendor/github.com/osbuild/images/pkg/upload/azure/azure.go @@ -10,14 +10,21 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" - "github.com/osbuild/osbuild-composer/internal/common" - "github.com/osbuild/osbuild-composer/internal/target" + "github.com/osbuild/images/internal/common" +) + +type HyperVGenerationType string + +const ( + HyperVGenV1 HyperVGenerationType = "V1" + HyperVGenV2 HyperVGenerationType = "V2" ) type Client struct { creds *azidentity.ClientSecretCredential resFact *armresources.ClientFactory storFact *armstorage.ClientFactory + compFact *armcompute.ClientFactory } // NewClient creates a client for accessing the Azure API. @@ -26,23 +33,29 @@ type Client struct { func NewClient(credentials Credentials, tenantID, subscriptionID string) (*Client, error) { creds, err := azidentity.NewClientSecretCredential(tenantID, credentials.clientID, credentials.clientSecret, nil) if err != nil { - return nil, fmt.Errorf("creating azure ClientSecretCredential failed: %v", err) + return nil, fmt.Errorf("creating azure ClientSecretCredential failed: %w", err) } resFact, err := armresources.NewClientFactory(subscriptionID, creds, nil) if err != nil { - return nil, fmt.Errorf("creating resources client factory failed: %v", err) + return nil, fmt.Errorf("creating resources client factory failed: %w", err) } storFact, err := armstorage.NewClientFactory(subscriptionID, creds, nil) if err != nil { - return nil, fmt.Errorf("creating storage client factory failed: %v", err) + return nil, fmt.Errorf("creating storage client factory failed: %w", err) + } + + compFact, err := armcompute.NewClientFactory(subscriptionID, creds, nil) + if err != nil { + return nil, fmt.Errorf("creating compute client factory failed: %w", err) } return &Client{ creds, resFact, storFact, + compFact, }, nil } @@ -65,7 +78,7 @@ func (ac Client) GetResourceNameByTag(ctx context.Context, resourceGroup string, result, err := pager.NextPage(ctx) if err != nil { - return "", fmt.Errorf("listing resources failed: %v", err) + return "", fmt.Errorf("listing resources failed: %w", err) } if len(result.Value) < 1 { @@ -80,7 +93,7 @@ func (ac Client) GetResourceGroupLocation(ctx context.Context, resourceGroup str group, err := c.Get(ctx, resourceGroup, nil) if err != nil { - return "", fmt.Errorf("retrieving resource group failed: %v", err) + return "", fmt.Errorf("retrieving resource group failed: %w", err) } return *group.Location, nil @@ -98,7 +111,7 @@ func (ac Client) CreateStorageAccount(ctx context.Context, resourceGroup, name, if location == "" { location, err = ac.GetResourceGroupLocation(ctx, resourceGroup) if err != nil { - return fmt.Errorf("retrieving resource group location failed: %v", err) + return fmt.Errorf("retrieving resource group location failed: %w", err) } } @@ -117,12 +130,12 @@ func (ac Client) CreateStorageAccount(ctx context.Context, resourceGroup, name, }, }, nil) if err != nil { - return fmt.Errorf("sending the create storage account request failed: %v", err) + return fmt.Errorf("sending the create storage account request failed: %w", err) } _, err = poller.PollUntilDone(ctx, nil) if err != nil { - return fmt.Errorf("create storage account request failed: %v", err) + return fmt.Errorf("create storage account request failed: %w", err) } return nil @@ -135,7 +148,7 @@ func (ac Client) GetStorageAccountKey(ctx context.Context, resourceGroup string, c := ac.storFact.NewAccountsClient() keys, err := c.ListKeys(ctx, resourceGroup, storageAccount, nil) if err != nil { - return "", fmt.Errorf("retrieving keys for a storage account failed: %v", err) + return "", fmt.Errorf("retrieving keys for a storage account failed: %w", err) } if len(keys.Keys) == 0 { @@ -148,26 +161,23 @@ func (ac Client) GetStorageAccountKey(ctx context.Context, resourceGroup string, // RegisterImage creates a generalized V1 Linux image from a given blob. // The location is optional and if not provided, it is determined // from the resource group. -func (ac Client) RegisterImage(ctx context.Context, subscriptionID, resourceGroup, storageAccount, storageContainer, blobName, imageName, location string, hyperVGen target.HyperVGenerationType) error { - c, err := armcompute.NewImagesClient(subscriptionID, ac.creds, nil) - if err != nil { - return fmt.Errorf("unable to create compute client: %v", err) - } - +func (ac Client) RegisterImage(ctx context.Context, resourceGroup, storageAccount, storageContainer, blobName, imageName, location string, hyperVGen HyperVGenerationType) error { + c := ac.compFact.NewImagesClient() blobURI := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", storageAccount, storageContainer, blobName) + var err error if location == "" { location, err = ac.GetResourceGroupLocation(ctx, resourceGroup) if err != nil { - return fmt.Errorf("retrieving resource group location failed: %v", err) + return fmt.Errorf("retrieving resource group location failed: %w", err) } } var hypvgen armcompute.HyperVGenerationTypes switch hyperVGen { - case target.HyperVGenV1: + case HyperVGenV1: hypvgen = armcompute.HyperVGenerationTypes(armcompute.HyperVGenerationTypesV1) - case target.HyperVGenV2: + case HyperVGenV2: hypvgen = armcompute.HyperVGenerationTypes(armcompute.HyperVGenerationTypesV2) default: return fmt.Errorf("Unknown hyper v generation type %v", hyperVGen) @@ -188,12 +198,12 @@ func (ac Client) RegisterImage(ctx context.Context, subscriptionID, resourceGrou Location: &location, }, nil) if err != nil { - return fmt.Errorf("sending the create image request failed: %v", err) + return fmt.Errorf("sending the create image request failed: %w", err) } _, err = imageFuture.PollUntilDone(ctx, nil) if err != nil { - return fmt.Errorf("create image request failed: %v", err) + return fmt.Errorf("create image request failed: %w", err) } return nil diff --git a/internal/upload/azure/azurestorage.go b/vendor/github.com/osbuild/images/pkg/upload/azure/azurestorage.go similarity index 93% rename from internal/upload/azure/azurestorage.go rename to vendor/github.com/osbuild/images/pkg/upload/azure/azurestorage.go index c0970e716..7479c60d1 100644 --- a/internal/upload/azure/azurestorage.go +++ b/vendor/github.com/osbuild/images/pkg/upload/azure/azurestorage.go @@ -23,7 +23,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/pageblob" "github.com/google/uuid" - "github.com/osbuild/osbuild-composer/internal/common" + "github.com/osbuild/images/internal/common" + "github.com/osbuild/images/pkg/datasizes" ) // StorageClient is a client for the Azure Storage API, @@ -39,7 +40,7 @@ type StorageClient struct { func NewStorageClient(storageAccount, storageAccessKey string) (*StorageClient, error) { credential, err := azblob.NewSharedKeyCredential(storageAccount, storageAccessKey) if err != nil { - return nil, fmt.Errorf("cannot create shared key credential: %v", err) + return nil, fmt.Errorf("cannot create shared key credential: %w", err) } return &StorageClient{ @@ -60,7 +61,7 @@ const DefaultUploadThreads = 16 // PageBlobMaxUploadPagesBytes defines how much bytes can we upload in a single UploadPages call. // See https://learn.microsoft.com/en-us/rest/api/storageservices/put-page -const PageBlobMaxUploadPagesBytes = 4 * 1024 * 1024 +const PageBlobMaxUploadPagesBytes = 4 * datasizes.MiB // allZerosSlice returns true if all values in the slice are equal to 0 func allZerosSlice(slice []byte) bool { @@ -93,14 +94,14 @@ func (c StorageClient) UploadPageBlob(metadata BlobMetadata, fileName string, th // Open the image file for reading imageFile, err := os.Open(fileName) if err != nil { - return fmt.Errorf("cannot open the image: %v", err) + return fmt.Errorf("cannot open the image: %w", err) } defer imageFile.Close() // Stat image to get the file size stat, err := imageFile.Stat() if err != nil { - return fmt.Errorf("cannot stat the image: %v", err) + return fmt.Errorf("cannot stat the image: %w", err) } if stat.Size()%512 != 0 { @@ -112,11 +113,11 @@ func (c StorageClient) UploadPageBlob(metadata BlobMetadata, fileName string, th /* #nosec G401 */ imageFileHash := md5.New() if _, err := io.Copy(imageFileHash, imageFile); err != nil { - return fmt.Errorf("cannot create md5 of the image: %v", err) + return fmt.Errorf("cannot create md5 of the image: %w", err) } // Move the cursor back to the start of the imageFile if _, err := imageFile.Seek(0, 0); err != nil { - return fmt.Errorf("cannot seek the image: %v", err) + return fmt.Errorf("cannot seek the image: %w", err) } // Create page blob. Page blob is required for VM images @@ -148,7 +149,7 @@ func (c StorageClient) UploadPageBlob(metadata BlobMetadata, fileName string, th if err == io.EOF { run = false } else { - return fmt.Errorf("reading the image failed: %v", err) + return fmt.Errorf("reading the image failed: %w", err) } } if n == 0 { @@ -173,7 +174,7 @@ func (c StorageClient) UploadPageBlob(metadata BlobMetadata, fileName string, th } _, err := client.UploadPages(ctx, common.NopSeekCloser(bytes.NewReader(buffer[:n])), uploadRange, nil) if err != nil { - err = fmt.Errorf("uploading a page failed: %v", err) + err = fmt.Errorf("uploading a page failed: %w", err) // Send the error to the error channel in a non-blocking way. If there is already an error, just discard this one select { case errorInGoroutine <- err: @@ -242,7 +243,7 @@ func (c StorageClient) TagBlob(ctx context.Context, metadata BlobMetadata, tags _, err = client.SetTags(ctx, tags, nil) if err != nil { - return fmt.Errorf("cannot tag the blob: %v", err) + return fmt.Errorf("cannot tag the blob: %w", err) } return nil diff --git a/internal/upload/azure/credentials.go b/vendor/github.com/osbuild/images/pkg/upload/azure/credentials.go similarity index 92% rename from internal/upload/azure/credentials.go rename to vendor/github.com/osbuild/images/pkg/upload/azure/credentials.go index 8bc0227bd..752bf5a06 100644 --- a/internal/upload/azure/credentials.go +++ b/vendor/github.com/osbuild/images/pkg/upload/azure/credentials.go @@ -25,7 +25,7 @@ func ParseAzureCredentialsFile(filename string) (*Credentials, error) { } _, err := toml.DecodeFile(filename, &creds) if err != nil { - return nil, fmt.Errorf("cannot parse azure credentials: %v", err) + return nil, fmt.Errorf("cannot parse azure credentials: %w", err) } return &Credentials{ diff --git a/vendor/modules.txt b/vendor/modules.txt index 15c5b1b4e..fded5c1e9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1110,6 +1110,7 @@ github.com/osbuild/images/pkg/rhsm/facts github.com/osbuild/images/pkg/rpmmd github.com/osbuild/images/pkg/runner github.com/osbuild/images/pkg/sbom +github.com/osbuild/images/pkg/upload/azure github.com/osbuild/images/pkg/upload/koji # github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d ## explicit; go 1.21