particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

many: switch to osbuild/images/pkg/upload for azure

Browse source 
This is part of consolidating all the upload code in images.
This commit is contained in:
Sanne Raymaekers 2025-07-29 11:13:50 +02:00 • committed by Achilleas Koutsou
parent 26ab15b1c9
commit 0e2daa201f
11 changed files with 59 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/osbuild-upload-azure/main.go
View file

@ -8,7 +8,7 @@ import (
"path"
"strings"
"github.com/osbuild/osbuild-composer/internal/upload/azure"
"github.com/osbuild/images/pkg/upload/azure"
)
func checkStringNotEmpty(variable string, errorMessage string) {

2
cmd/osbuild-worker/config.go
View file

@ -6,7 +6,7 @@ import (
"time"
"github.com/BurntSushi/toml"
"github.com/osbuild/osbuild-composer/internal/upload/azure"
"github.com/osbuild/images/pkg/upload/azure"
"github.com/sirupsen/logrus"
)

9
cmd/osbuild-worker/jobimpl-osbuild.go
View file

@ -29,12 +29,12 @@ import (
"github.com/google/uuid"
"github.com/sirupsen/logrus"
"github.com/osbuild/images/pkg/upload/azure"
"github.com/osbuild/images/pkg/upload/koji"
"github.com/osbuild/osbuild-composer/internal/cloud/awscloud"
"github.com/osbuild/osbuild-composer/internal/cloud/gcp"
"github.com/osbuild/osbuild-composer/internal/osbuildexecutor"
"github.com/osbuild/osbuild-composer/internal/target"
"github.com/osbuild/osbuild-composer/internal/upload/azure"
"github.com/osbuild/osbuild-composer/internal/upload/vmware"
"github.com/osbuild/osbuild-composer/internal/worker"
"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
@ -978,16 +978,19 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error {
}
logWithId.Info("[Azure] 📝 Registering the image")
hyperVGen := azure.HyperVGenV1
if targetOptions.HyperVGeneration == target.HyperVGenV2 {
hyperVGen = azure.HyperVGenV2
}
err = c.RegisterImage(
ctx,
targetOptions.SubscriptionID,
targetOptions.ResourceGroup,
storageAccount,
storageContainer,
blobName,
jobTarget.ImageName,
location,
targetOptions.HyperVGeneration,
hyperVGen,
)
if err != nil {
targetResult.TargetError = clienterrors.New(clienterrors.ErrorImportingImage, fmt.Sprintf("registering the image failed: %v", err), nil)

2
cmd/osbuild-worker/main.go
View file

@ -21,9 +21,9 @@ import (
"github.com/osbuild/images/pkg/arch"
"github.com/osbuild/images/pkg/dnfjson"
"github.com/osbuild/images/pkg/upload/azure"
"github.com/osbuild/images/pkg/upload/koji"
"github.com/osbuild/osbuild-composer/internal/cloud/awscloud"
"github.com/osbuild/osbuild-composer/internal/upload/azure"
"github.com/osbuild/osbuild-composer/internal/upload/oci"
"github.com/osbuild/osbuild-composer/internal/worker"
)

8
go.mod
View file

@ -8,10 +8,6 @@ require (
cloud.google.com/go/compute v1.40.0
cloud.google.com/go/storage v1.55.0
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1
github.com/Azure/go-autorest/autorest v0.11.30
github.com/Azure/go-autorest/autorest/azure/auth v0.5.13
@ -73,7 +69,11 @@ require (
cloud.google.com/go/monitoring v1.24.2 // indirect
dario.cat/mergo v1.0.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.7.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect
github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect

2
internal/boot/azuretest/azure.go
View file

@ -21,7 +21,7 @@ import (
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/upload/azure"
"github.com/osbuild/images/pkg/upload/azure"
)
// wrapErrorf returns error constructed using fmt.Errorf from format and any

33
internal/upload/azure/azurestorage_test.go
View file

@ -1,33 +0,0 @@
package azure
import (
"regexp"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRandomStorageAccountName(t *testing.T) {
randomName := RandomStorageAccountName("ib")
assert.Len(t, randomName, 24)
r := regexp.MustCompile(`^[\d\w]{24}$`)
assert.True(t, r.MatchString(randomName), "the returned name should be 24 characters long and contain only alphanumerical characters")
}
func TestEnsureVHDExtension(t *testing.T) {
tests := []struct {
s string
want string
}{
{s: "toucan.zip", want: "toucan.zip.vhd"},
{s: "kingfisher.vhd", want: "kingfisher.vhd"},
}
for _, tt := range tests {
t.Run(tt.s, func(t *testing.T) {
require.Equal(t, tt.want, EnsureVHDExtension(tt.s))
})
}
}

54
internal/upload/azure/azure.go → vendor/github.com/osbuild/images/pkg/upload/azure/azure.go generated vendored
View file

@ -10,14 +10,21 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/target"
"github.com/osbuild/images/internal/common"
)
type HyperVGenerationType string
const (
HyperVGenV1 HyperVGenerationType = "V1"
HyperVGenV2 HyperVGenerationType = "V2"
)
type Client struct {
creds *azidentity.ClientSecretCredential
resFact *armresources.ClientFactory
storFact *armstorage.ClientFactory
compFact *armcompute.ClientFactory
}
// NewClient creates a client for accessing the Azure API.
@ -26,23 +33,29 @@ type Client struct {
func NewClient(credentials Credentials, tenantID, subscriptionID string) (*Client, error) {
creds, err := azidentity.NewClientSecretCredential(tenantID, credentials.clientID, credentials.clientSecret, nil)
if err != nil {
return nil, fmt.Errorf("creating azure ClientSecretCredential failed: %v", err)
return nil, fmt.Errorf("creating azure ClientSecretCredential failed: %w", err)
}
resFact, err := armresources.NewClientFactory(subscriptionID, creds, nil)
if err != nil {
return nil, fmt.Errorf("creating resources client factory failed: %v", err)
return nil, fmt.Errorf("creating resources client factory failed: %w", err)
}
storFact, err := armstorage.NewClientFactory(subscriptionID, creds, nil)
if err != nil {
return nil, fmt.Errorf("creating storage client factory failed: %v", err)
return nil, fmt.Errorf("creating storage client factory failed: %w", err)
}
compFact, err := armcompute.NewClientFactory(subscriptionID, creds, nil)
if err != nil {
return nil, fmt.Errorf("creating compute client factory failed: %w", err)
}
return &Client{
creds,
resFact,
storFact,
compFact,
}, nil
}
@ -65,7 +78,7 @@ func (ac Client) GetResourceNameByTag(ctx context.Context, resourceGroup string,
result, err := pager.NextPage(ctx)
if err != nil {
return "", fmt.Errorf("listing resources failed: %v", err)
return "", fmt.Errorf("listing resources failed: %w", err)
}
if len(result.Value) < 1 {
@ -80,7 +93,7 @@ func (ac Client) GetResourceGroupLocation(ctx context.Context, resourceGroup str
group, err := c.Get(ctx, resourceGroup, nil)
if err != nil {
return "", fmt.Errorf("retrieving resource group failed: %v", err)
return "", fmt.Errorf("retrieving resource group failed: %w", err)
}
return *group.Location, nil
@ -98,7 +111,7 @@ func (ac Client) CreateStorageAccount(ctx context.Context, resourceGroup, name,
if location == "" {
location, err = ac.GetResourceGroupLocation(ctx, resourceGroup)
if err != nil {
return fmt.Errorf("retrieving resource group location failed: %v", err)
return fmt.Errorf("retrieving resource group location failed: %w", err)
}
}
@ -117,12 +130,12 @@ func (ac Client) CreateStorageAccount(ctx context.Context, resourceGroup, name,
},
}, nil)
if err != nil {
return fmt.Errorf("sending the create storage account request failed: %v", err)
return fmt.Errorf("sending the create storage account request failed: %w", err)
}
_, err = poller.PollUntilDone(ctx, nil)
if err != nil {
return fmt.Errorf("create storage account request failed: %v", err)
return fmt.Errorf("create storage account request failed: %w", err)
}
return nil
@ -135,7 +148,7 @@ func (ac Client) GetStorageAccountKey(ctx context.Context, resourceGroup string,
c := ac.storFact.NewAccountsClient()
keys, err := c.ListKeys(ctx, resourceGroup, storageAccount, nil)
if err != nil {
return "", fmt.Errorf("retrieving keys for a storage account failed: %v", err)
return "", fmt.Errorf("retrieving keys for a storage account failed: %w", err)
}
if len(keys.Keys) == 0 {
@ -148,26 +161,23 @@ func (ac Client) GetStorageAccountKey(ctx context.Context, resourceGroup string,
// RegisterImage creates a generalized V1 Linux image from a given blob.
// The location is optional and if not provided, it is determined
// from the resource group.
func (ac Client) RegisterImage(ctx context.Context, subscriptionID, resourceGroup, storageAccount, storageContainer, blobName, imageName, location string, hyperVGen target.HyperVGenerationType) error {
c, err := armcompute.NewImagesClient(subscriptionID, ac.creds, nil)
if err != nil {
return fmt.Errorf("unable to create compute client: %v", err)
}
func (ac Client) RegisterImage(ctx context.Context, resourceGroup, storageAccount, storageContainer, blobName, imageName, location string, hyperVGen HyperVGenerationType) error {
c := ac.compFact.NewImagesClient()
blobURI := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", storageAccount, storageContainer, blobName)
var err error
if location == "" {
location, err = ac.GetResourceGroupLocation(ctx, resourceGroup)
if err != nil {
return fmt.Errorf("retrieving resource group location failed: %v", err)
return fmt.Errorf("retrieving resource group location failed: %w", err)
}
}
var hypvgen armcompute.HyperVGenerationTypes
switch hyperVGen {
case target.HyperVGenV1:
case HyperVGenV1:
hypvgen = armcompute.HyperVGenerationTypes(armcompute.HyperVGenerationTypesV1)
case target.HyperVGenV2:
case HyperVGenV2:
hypvgen = armcompute.HyperVGenerationTypes(armcompute.HyperVGenerationTypesV2)
default:
return fmt.Errorf("Unknown hyper v generation type %v", hyperVGen)
@ -188,12 +198,12 @@ func (ac Client) RegisterImage(ctx context.Context, subscriptionID, resourceGrou
Location: &location,
}, nil)
if err != nil {
return fmt.Errorf("sending the create image request failed: %v", err)
return fmt.Errorf("sending the create image request failed: %w", err)
}
_, err = imageFuture.PollUntilDone(ctx, nil)
if err != nil {
return fmt.Errorf("create image request failed: %v", err)
return fmt.Errorf("create image request failed: %w", err)
}
return nil

21
internal/upload/azure/azurestorage.go → vendor/github.com/osbuild/images/pkg/upload/azure/azurestorage.go generated vendored
View file

@ -23,7 +23,8 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/pageblob"
"github.com/google/uuid"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/images/internal/common"
"github.com/osbuild/images/pkg/datasizes"
)
// StorageClient is a client for the Azure Storage API,
@ -39,7 +40,7 @@ type StorageClient struct {
func NewStorageClient(storageAccount, storageAccessKey string) (*StorageClient, error) {
credential, err := azblob.NewSharedKeyCredential(storageAccount, storageAccessKey)
if err != nil {
return nil, fmt.Errorf("cannot create shared key credential: %v", err)
return nil, fmt.Errorf("cannot create shared key credential: %w", err)
}
return &StorageClient{
@ -60,7 +61,7 @@ const DefaultUploadThreads = 16
// PageBlobMaxUploadPagesBytes defines how much bytes can we upload in a single UploadPages call.
// See https://learn.microsoft.com/en-us/rest/api/storageservices/put-page
const PageBlobMaxUploadPagesBytes = 4 * 1024 * 1024
const PageBlobMaxUploadPagesBytes = 4 * datasizes.MiB
// allZerosSlice returns true if all values in the slice are equal to 0
func allZerosSlice(slice []byte) bool {
@ -93,14 +94,14 @@ func (c StorageClient) UploadPageBlob(metadata BlobMetadata, fileName string, th
// Open the image file for reading
imageFile, err := os.Open(fileName)
if err != nil {
return fmt.Errorf("cannot open the image: %v", err)
return fmt.Errorf("cannot open the image: %w", err)
}
defer imageFile.Close()
// Stat image to get the file size
stat, err := imageFile.Stat()
if err != nil {
return fmt.Errorf("cannot stat the image: %v", err)
return fmt.Errorf("cannot stat the image: %w", err)
}
if stat.Size()%512 != 0 {
@ -112,11 +113,11 @@ func (c StorageClient) UploadPageBlob(metadata BlobMetadata, fileName string, th
/* #nosec G401 */
imageFileHash := md5.New()
if _, err := io.Copy(imageFileHash, imageFile); err != nil {
return fmt.Errorf("cannot create md5 of the image: %v", err)
return fmt.Errorf("cannot create md5 of the image: %w", err)
}
// Move the cursor back to the start of the imageFile
if _, err := imageFile.Seek(0, 0); err != nil {
return fmt.Errorf("cannot seek the image: %v", err)
return fmt.Errorf("cannot seek the image: %w", err)
}
// Create page blob. Page blob is required for VM images
@ -148,7 +149,7 @@ func (c StorageClient) UploadPageBlob(metadata BlobMetadata, fileName string, th
if err == io.EOF {
run = false
} else {
return fmt.Errorf("reading the image failed: %v", err)
return fmt.Errorf("reading the image failed: %w", err)
}
}
if n == 0 {
@ -173,7 +174,7 @@ func (c StorageClient) UploadPageBlob(metadata BlobMetadata, fileName string, th
}
_, err := client.UploadPages(ctx, common.NopSeekCloser(bytes.NewReader(buffer[:n])), uploadRange, nil)
if err != nil {
err = fmt.Errorf("uploading a page failed: %v", err)
err = fmt.Errorf("uploading a page failed: %w", err)
// Send the error to the error channel in a non-blocking way. If there is already an error, just discard this one
select {
case errorInGoroutine <- err:
@ -242,7 +243,7 @@ func (c StorageClient) TagBlob(ctx context.Context, metadata BlobMetadata, tags
_, err = client.SetTags(ctx, tags, nil)
if err != nil {
return fmt.Errorf("cannot tag the blob: %v", err)
return fmt.Errorf("cannot tag the blob: %w", err)
}
return nil

2
internal/upload/azure/credentials.go → vendor/github.com/osbuild/images/pkg/upload/azure/credentials.go generated vendored
View file

@ -25,7 +25,7 @@ func ParseAzureCredentialsFile(filename string) (*Credentials, error) {
}
_, err := toml.DecodeFile(filename, &creds)
if err != nil {
return nil, fmt.Errorf("cannot parse azure credentials: %v", err)
return nil, fmt.Errorf("cannot parse azure credentials: %w", err)
}
return &Credentials{

1
vendor/modules.txt vendored
View file

@ -1110,6 +1110,7 @@ github.com/osbuild/images/pkg/rhsm/facts
github.com/osbuild/images/pkg/rpmmd
github.com/osbuild/images/pkg/runner
github.com/osbuild/images/pkg/sbom
github.com/osbuild/images/pkg/upload/azure
github.com/osbuild/images/pkg/upload/koji
# github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d
## explicit; go 1.21