worker: add azure image upload target

This commit adds and implements org.osbuild.azure.image target.

Let's talk about the already implemented org.osbuild.azure target firstly:
The purpose of this target is to authenticate using the Azure Storage
credentials and upload the image file as a Page Blob. Page Blob is basically
an object in storage and it cannot be directly used to launch a VM. To achieve
that, you need to define an actual Azure Image with the Page Blob attached.

For the cloud API, we would like to create an actual Azure Image that is
immediately available for new VMs. The new target accomplishes it.
To achieve this, it must use a different authentication method: Azure OAuth.
The other important difference is that currently, the credentials are stored
on the worker and not in target options. This should lead to better security
because we don't send the credentials over network. In the future, we would
like to have credential-less setup using workers in Azure with the right
IAM policies applied but this requires more investigation and is not
implemented in this commit.

Signed-off-by: Ondřej Budai <ondrej@budai.cz>

internal/upload/azure/azure.go

@@ -0,0 +1,150 @@
+package azure
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/compute/mgmt/compute"
+	"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/resources/mgmt/resources"
+	"github.com/Azure/azure-sdk-for-go/profiles/2019-03-01/storage/mgmt/storage"
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure/auth"
+)
+
+type Client struct {
+	authorizer autorest.Authorizer
+}
+
+// NewClient creates a client for accessing the Azure API.
+// See https://docs.microsoft.com/en-us/rest/api/azure/
+// If you need to work with the Azure Storage API, see NewStorageClient
+func NewClient(credentials Credentials, tenantID string) (*Client, error) {
+	credentialsConfig := auth.NewClientCredentialsConfig(credentials.clientID, credentials.clientSecret, tenantID)
+	authorizer, err := credentialsConfig.Authorizer()
+	if err != nil {
+		return nil, fmt.Errorf("creating an azure authorizer failed: %v", err)
+	}
+
+	return &Client{
+		authorizer: authorizer,
+	}, nil
+}
+
+// Tag is a name-value pair representing the tag structure in Azure
+type Tag struct {
+	Name  string
+	Value string
+}
+
+// GetResourceNameByTag returns a name of an Azure resource tagged with the
+// given `tag`. Note that if multiple resources with the same tag exists
+// in the specified resource group, only one name is returned. It's undefined
+// which one it is.
+func (ac Client) GetResourceNameByTag(ctx context.Context, subscriptionID, resourceGroup string, tag Tag) (string, error) {
+	c := resources.NewClient(subscriptionID)
+	c.Authorizer = ac.authorizer
+
+	filter := fmt.Sprintf("tagName eq '%s' and tagValue eq '%s'", tag.Name, tag.Value)
+	result, err := c.ListByResourceGroup(ctx, resourceGroup, filter, "", nil)
+	if err != nil {
+		return "", fmt.Errorf("listing resources failed: %v", err)
+	}
+
+	if len(result.Values()) < 1 {
+		return "", nil
+	}
+
+	return *result.Values()[0].Name, nil
+}
+
+// CreateStorageAccount creates a storage account in the specified resource
+// group. The location parameter can be used to specify its location. The tag
+// can be used to specify a tag attached to the account.
+func (ac Client) CreateStorageAccount(ctx context.Context, subscriptionID, resourceGroup, name, location string, tag Tag) error {
+	c := storage.NewAccountsClient(subscriptionID)
+	c.Authorizer = ac.authorizer
+
+	result, err := c.Create(ctx, resourceGroup, name, storage.AccountCreateParameters{
+		Sku: &storage.Sku{
+			Name: storage.StandardRAGRS, // TODO: investigate the default value
+			Tier: storage.Standard,
+		},
+		Location: &location,
+		Tags: map[string]*string{
+			tag.Name: &tag.Value,
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("sending the create storage account request failed: %v", err)
+	}
+
+	err = result.WaitForCompletionRef(ctx, c.Client)
+	if err != nil {
+		return fmt.Errorf("waiting for the create storage account request failed: %v", err)
+	}
+
+	_, err = result.Result(c)
+	if err != nil {
+		return fmt.Errorf("create storage account request failed: %v", err)
+	}
+
+	return nil
+}
+
+// GetStorageAccountKey returns a storage account key that can be used to
+// access the given storage account. This method always returns only the first
+// key.
+func (ac Client) GetStorageAccountKey(ctx context.Context, subscriptionID, resourceGroup string, storageAccount string) (string, error) {
+	c := storage.NewAccountsClient(subscriptionID)
+	c.Authorizer = ac.authorizer
+
+	keys, err := c.ListKeys(ctx, resourceGroup, storageAccount)
+	if err != nil {
+		return "", fmt.Errorf("retrieving keys for a storage account failed: %v", err)
+	}
+
+	if len(*keys.Keys) == 0 {
+		return "", errors.New("azure returned an empty list of keys")
+	}
+
+	return *(*keys.Keys)[0].Value, nil
+}
+
+// RegisterImage creates a generalized V1 Linux image from a given blob.
+func (ac Client) RegisterImage(ctx context.Context, subscriptionID, resourceGroup, storageAccount, storageContainer, blobName, imageName, location string) error {
+	c := compute.NewImagesClient(subscriptionID)
+	c.Authorizer = ac.authorizer
+
+	blobURI := fmt.Sprintf("https://%s.blob.core.windows.net/%s/%s", storageAccount, storageContainer, blobName)
+
+	imageFuture, err := c.CreateOrUpdate(ctx, resourceGroup, imageName, compute.Image{
+		Response: autorest.Response{},
+		ImageProperties: &compute.ImageProperties{
+			SourceVirtualMachine: nil,
+			StorageProfile: &compute.ImageStorageProfile{
+				OsDisk: &compute.ImageOSDisk{
+					OsType:  compute.Linux,
+					BlobURI: &blobURI,
+					OsState: compute.Generalized,
+				},
+			},
+		},
+		Location: &location,
+	})
+	if err != nil {
+		return fmt.Errorf("sending the create image request failed: %v", err)
+	}
+
+	err = imageFuture.WaitForCompletionRef(ctx, c.Client)
+	if err != nil {
+		return fmt.Errorf("waiting for the create image request failed: %v", err)
+	}
+
+	_, err = imageFuture.Result(c)
+	if err != nil {
+		return fmt.Errorf("create image request failed: %v", err)
+	}
+
+	return nil
+}

internal/upload/azure/azurestorage.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/Azure/azure-pipeline-go/pipeline"
 	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/google/uuid"
 )
 
 // StorageClient is a client for the Azure Storage API,
@@ -47,6 +48,9 @@ type BlobMetadata struct {
 	BlobName       string
 }
 
+// DefaultUploadThreads defines a tested default value for the UploadPageBlob method's threads parameter.
+const DefaultUploadThreads = 16
+
 // UploadPageBlob takes the metadata and credentials required to upload the image specified by `fileName`
 // It can speed up the upload by using goroutines. The number of parallel goroutines is bounded by
 // the `threads` argument.
@@ -169,3 +173,31 @@ func (c StorageClient) UploadPageBlob(metadata BlobMetadata, fileName string, th
 
 	return nil
 }
+
+// CreateStorageContainerIfNotExist creates an empty storage container inside
+// a storage account. If a container with the same name already exists,
+// this method is no-op.
+func (c StorageClient) CreateStorageContainerIfNotExist(ctx context.Context, storageAccount, name string) error {
+	URL, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", storageAccount, name))
+	containerURL := azblob.NewContainerURL(*URL, c.pipeline)
+
+	_, err := containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
+	if err != nil {
+		if storageErr, ok := err.(azblob.StorageError); ok && storageErr.(azblob.StorageError).ServiceCode() == azblob.ServiceCodeContainerAlreadyExists {
+			return nil
+		}
+		return fmt.Errorf("cannot create a storage container: %v", err)
+	}
+
+	return nil
+}
+
+// RandomStorageAccountName returns a randomly generated name that can be used
+// for a storage account. This means that it must use only alphanumeric
+// characters and its length must be 24 or lower.
+func RandomStorageAccountName(prefix string) string {
+	id := uuid.New().String()
+	id = strings.ReplaceAll(id, "-", "")
+
+	return (prefix + id)[:24]
+}

internal/upload/azure/azurestorage_test.go

@@ -0,0 +1,17 @@
+package azure
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRandomStorageAccountName(t *testing.T) {
+	randomName := RandomStorageAccountName("ib")
+
+	assert.Len(t, randomName, 24)
+
+	r := regexp.MustCompile(`^[\d\w]{24}$`)
+	assert.True(t, r.MatchString(randomName), "the returned name should be 24 characters long and contain only alphanumerical characters")
+}

internal/upload/azure/credentials.go

@@ -0,0 +1,35 @@
+package azure
+
+import (
+	"fmt"
+
+	"github.com/BurntSushi/toml"
+)
+
+type Credentials struct {
+	clientID     string
+	clientSecret string
+}
+
+// ParseAzureCredentialsFile parses a credentials file for azure.
+// The file is in toml format and contains two keys: client_id and
+// client_secret
+//
+// Example of the file:
+// client_id     = "clientIdOfMyApplication"
+// client_secret = "ToucanToucan~"
+func ParseAzureCredentialsFile(filename string) (*Credentials, error) {
+	var creds struct {
+		ClientID     string `toml:"client_id"`
+		ClientSecret string `toml:"client_secret"`
+	}
+	_, err := toml.DecodeFile(filename, &creds)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse azure credentials: %v", err)
+	}
+
+	return &Credentials{
+		clientID:     creds.ClientID,
+		clientSecret: creds.ClientSecret,
+	}, nil
+}