worker: allow configuring number of upload threads for Azure

The default number of threads (16) is OK for general use case. However,
we are being asked by RH IT to lower the number of threads when
uploading the image to Azure using proxy server.

Make the number of threads configurable in the worker configuration and
default to the currently used value if it is not provided.

Signed-off-by: Tomáš Hozza <thozza@redhat.com>

cmd/osbuild-worker/config.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/BurntSushi/toml"
+	"github.com/osbuild/osbuild-composer/internal/upload/azure"
 	"github.com/sirupsen/logrus"
 )
 
@@ -27,7 +29,8 @@ type gcpConfig struct {
 }
 
 type azureConfig struct {
-	Credentials string `toml:"credentials"`
+	Credentials   string `toml:"credentials"`
+	UploadThreads int    `toml:"upload_threads"`
 }
 
 type awsConfig struct {
@@ -90,5 +93,14 @@ func parseConfig(file string) (*workerConfig, error) {
 		logrus.Info("Configuration file not found, using defaults")
 	}
 
+	// set defaults for Azure only if the config section is present
+	if config.Azure != nil {
+		if config.Azure.UploadThreads == 0 {
+			config.Azure.UploadThreads = azure.DefaultUploadThreads
+		} else if config.Azure.UploadThreads < 0 {
+			return nil, fmt.Errorf("invalid number of Azure upload threads: %d", config.Azure.UploadThreads)
+		}
+	}
+
 	return &config, nil
 }

cmd/osbuild-worker/config_test.go

@@ -43,6 +43,7 @@ credentials = "/etc/osbuild-worker/gcp-creds"
 
 [azure]
 credentials = "/etc/osbuild-worker/azure-creds"
+upload_threads = 8
 
 [aws]
 credentials = "/etc/osbuild-worker/aws-creds"
@@ -88,7 +89,8 @@ offline_token = "/etc/osbuild-worker/offline_token"
 					Credentials: "/etc/osbuild-worker/gcp-creds",
 				},
 				Azure: &azureConfig{
-					Credentials: "/etc/osbuild-worker/azure-creds",
+					Credentials:   "/etc/osbuild-worker/azure-creds",
+					UploadThreads: 8,
 				},
 				AWS: &awsConfig{
 					Credentials: "/etc/osbuild-worker/aws-creds",
@@ -141,6 +143,17 @@ offline_token = "/etc/osbuild-worker/offline_token"
 		_, err := parseConfig(configFile)
 		require.Error(t, err)
 	})
+
+	t.Run("wrong Azure config", func(t *testing.T) {
+		configFile := prepareConfig(t, `
+[azure]
+credentials = "/etc/osbuild-worker/azure-creds"
+upload_threads = -5
+`)
+		_, err := parseConfig(configFile)
+		require.Error(t, err)
+	})
+
 }
 
 func prepareConfig(t *testing.T, config string) string {

cmd/osbuild-worker/jobimpl-osbuild.go

@@ -53,12 +53,17 @@ type ContainersConfiguration struct {
 	TLSVerify    *bool
 }
 
+type AzureConfiguration struct {
+	Creds         *azure.Credentials
+	UploadThreads int
+}
+
 type OSBuildJobImpl struct {
 	Store            string
 	Output           string
 	KojiServers      map[string]kojiServer
 	GCPConfig        GCPConfiguration
-	AzureCreds       *azure.Credentials
+	AzureConfig      AzureConfiguration
 	AWSCreds         string
 	AWSBucket        string
 	S3Config         S3Configuration
@@ -639,12 +644,12 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error {
 			targetResult = target.NewAzureImageTargetResult(nil)
 			ctx := context.Background()
 
-			if impl.AzureCreds == nil {
+			if impl.AzureConfig.Creds == nil {
 				targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorSharingTarget, "osbuild job has org.osbuild.azure.image target but this worker doesn't have azure credentials", nil)
 				break
 			}
 
-			c, err := azure.NewClient(*impl.AzureCreds, targetOptions.TenantID)
+			c, err := azure.NewClient(*impl.AzureConfig.Creds, targetOptions.TenantID)
 			if err != nil {
 				targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, err.Error(), nil)
 				break
@@ -738,7 +743,7 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error {
 					BlobName:       blobName,
 				},
 				imagePath,
-				azure.DefaultUploadThreads,
+				impl.AzureConfig.UploadThreads,
 			)
 			if err != nil {
 				targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, fmt.Sprintf("uploading the image failed: %v", err), nil)

cmd/osbuild-worker/main.go

@@ -341,12 +341,13 @@ func main() {
 	// Load Azure credentials early. If the credentials file is malformed,
 	// we can report the issue early instead of waiting for the first osbuild
 	// job with the org.osbuild.azure.image target.
-	var azureCredentials *azure.Credentials
+	var azureConfig AzureConfiguration
 	if config.Azure != nil {
-		azureCredentials, err = azure.ParseAzureCredentialsFile(config.Azure.Credentials)
+		azureConfig.Creds, err = azure.ParseAzureCredentialsFile(config.Azure.Credentials)
 		if err != nil {
 			logrus.Fatalf("cannot load azure credentials: %v", err)
 		}
+		azureConfig.UploadThreads = config.Azure.UploadThreads
 	}
 
 	// If the credentials are not provided in the configuration, then the
@@ -437,7 +438,7 @@ func main() {
 			Output:      output,
 			KojiServers: kojiServers,
 			GCPConfig:   gcpConfig,
-			AzureCreds:  azureCredentials,
+			AzureConfig: azureConfig,
 			AWSCreds:    awsCredentials,
 			AWSBucket:   awsBucket,
 			S3Config: S3Configuration{