From 0e4a5b34b214a6496b22f808fae402f88bebe6a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= Date: Fri, 3 Mar 2023 14:35:36 +0100 Subject: [PATCH] worker: allow configuring number of upload threads for Azure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default number of threads (16) is OK for general use case. However, we are being asked by RH IT to lower the number of threads when uploading the image to Azure using proxy server. Make the number of threads configurable in the worker configuration and default to the currently used value if it is not provided. Signed-off-by: Tomáš Hozza --- cmd/osbuild-worker/config.go | 14 +++++++++++++- cmd/osbuild-worker/config_test.go | 15 ++++++++++++++- cmd/osbuild-worker/jobimpl-osbuild.go | 13 +++++++++---- cmd/osbuild-worker/main.go | 7 ++++--- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/cmd/osbuild-worker/config.go b/cmd/osbuild-worker/config.go index b968f687f..2f0d9b76a 100644 --- a/cmd/osbuild-worker/config.go +++ b/cmd/osbuild-worker/config.go @@ -1,9 +1,11 @@ package main import ( + "fmt" "os" "github.com/BurntSushi/toml" + "github.com/osbuild/osbuild-composer/internal/upload/azure" "github.com/sirupsen/logrus" ) @@ -27,7 +29,8 @@ type gcpConfig struct { } type azureConfig struct { - Credentials string `toml:"credentials"` + Credentials string `toml:"credentials"` + UploadThreads int `toml:"upload_threads"` } type awsConfig struct { @@ -90,5 +93,14 @@ func parseConfig(file string) (*workerConfig, error) { logrus.Info("Configuration file not found, using defaults") } + // set defaults for Azure only if the config section is present + if config.Azure != nil { + if config.Azure.UploadThreads == 0 { + config.Azure.UploadThreads = azure.DefaultUploadThreads + } else if config.Azure.UploadThreads < 0 { + return nil, fmt.Errorf("invalid number of Azure upload threads: %d", config.Azure.UploadThreads) + } + } + return &config, nil } diff --git a/cmd/osbuild-worker/config_test.go b/cmd/osbuild-worker/config_test.go index ebea302b8..aa2cb33b7 100644 --- a/cmd/osbuild-worker/config_test.go +++ b/cmd/osbuild-worker/config_test.go @@ -43,6 +43,7 @@ credentials = "/etc/osbuild-worker/gcp-creds" [azure] credentials = "/etc/osbuild-worker/azure-creds" +upload_threads = 8 [aws] credentials = "/etc/osbuild-worker/aws-creds" @@ -88,7 +89,8 @@ offline_token = "/etc/osbuild-worker/offline_token" Credentials: "/etc/osbuild-worker/gcp-creds", }, Azure: &azureConfig{ - Credentials: "/etc/osbuild-worker/azure-creds", + Credentials: "/etc/osbuild-worker/azure-creds", + UploadThreads: 8, }, AWS: &awsConfig{ Credentials: "/etc/osbuild-worker/aws-creds", @@ -141,6 +143,17 @@ offline_token = "/etc/osbuild-worker/offline_token" _, err := parseConfig(configFile) require.Error(t, err) }) + + t.Run("wrong Azure config", func(t *testing.T) { + configFile := prepareConfig(t, ` +[azure] +credentials = "/etc/osbuild-worker/azure-creds" +upload_threads = -5 +`) + _, err := parseConfig(configFile) + require.Error(t, err) + }) + } func prepareConfig(t *testing.T, config string) string { diff --git a/cmd/osbuild-worker/jobimpl-osbuild.go b/cmd/osbuild-worker/jobimpl-osbuild.go index 9c6124539..3114abd89 100644 --- a/cmd/osbuild-worker/jobimpl-osbuild.go +++ b/cmd/osbuild-worker/jobimpl-osbuild.go @@ -53,12 +53,17 @@ type ContainersConfiguration struct { TLSVerify *bool } +type AzureConfiguration struct { + Creds *azure.Credentials + UploadThreads int +} + type OSBuildJobImpl struct { Store string Output string KojiServers map[string]kojiServer GCPConfig GCPConfiguration - AzureCreds *azure.Credentials + AzureConfig AzureConfiguration AWSCreds string AWSBucket string S3Config S3Configuration @@ -639,12 +644,12 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { targetResult = target.NewAzureImageTargetResult(nil) ctx := context.Background() - if impl.AzureCreds == nil { + if impl.AzureConfig.Creds == nil { targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorSharingTarget, "osbuild job has org.osbuild.azure.image target but this worker doesn't have azure credentials", nil) break } - c, err := azure.NewClient(*impl.AzureCreds, targetOptions.TenantID) + c, err := azure.NewClient(*impl.AzureConfig.Creds, targetOptions.TenantID) if err != nil { targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorInvalidTargetConfig, err.Error(), nil) break @@ -738,7 +743,7 @@ func (impl *OSBuildJobImpl) Run(job worker.Job) error { BlobName: blobName, }, imagePath, - azure.DefaultUploadThreads, + impl.AzureConfig.UploadThreads, ) if err != nil { targetResult.TargetError = clienterrors.WorkerClientError(clienterrors.ErrorUploadingImage, fmt.Sprintf("uploading the image failed: %v", err), nil) diff --git a/cmd/osbuild-worker/main.go b/cmd/osbuild-worker/main.go index 51b989cf4..fa767db72 100644 --- a/cmd/osbuild-worker/main.go +++ b/cmd/osbuild-worker/main.go @@ -341,12 +341,13 @@ func main() { // Load Azure credentials early. If the credentials file is malformed, // we can report the issue early instead of waiting for the first osbuild // job with the org.osbuild.azure.image target. - var azureCredentials *azure.Credentials + var azureConfig AzureConfiguration if config.Azure != nil { - azureCredentials, err = azure.ParseAzureCredentialsFile(config.Azure.Credentials) + azureConfig.Creds, err = azure.ParseAzureCredentialsFile(config.Azure.Credentials) if err != nil { logrus.Fatalf("cannot load azure credentials: %v", err) } + azureConfig.UploadThreads = config.Azure.UploadThreads } // If the credentials are not provided in the configuration, then the @@ -437,7 +438,7 @@ func main() { Output: output, KojiServers: kojiServers, GCPConfig: gcpConfig, - AzureCreds: azureCredentials, + AzureConfig: azureConfig, AWSCreds: awsCredentials, AWSBucket: awsBucket, S3Config: S3Configuration{