particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support Generic S3 upload in Composer API

Browse source 
Use case
--------
If Endpoint is not set and Region is - upload to AWS S3
If both the Endpoint and Region are set - upload the Generic S3 via Weldr API
If neither the Endpoint and Region are set - upload the Generic S3 via Composer API (use configuration)

jobimpl-osbuild
---------------
Add configuration fields for Generic S3 upload
Support S3 upload requests coming from Weldr or Composer API to either AWS or Generic S3
Weldr API for Generic S3 requires that all connection parameters but the credentials be passed in the API call
Composer API for Generic S3 requires that all conneciton parameters are taken from the configuration
Adjust to the consolidation in Target and UploadOptions

Target and UploadOptions
------------------------
Add the fields that were specific to the Generic S3 structures to the AWS S3 one
Remove the structures for Generic S3 and always use the AWS S3 ones

Worker Main
-----------
Add Endpoint, Region, Bucket, CABundle and SkipSSLVerification to the configuration structure
Pass the values to the Server

Weldr API
---------
Keep the generic.s3 provider name to maintain the API, but unmarshel into awsS3UploadSettings

tests - api.sh
--------------
Allow the caller to specifiy either AWS or Generic S3 upload targets for specific image types
Implement the pieces required for testing upload to a Generic S3 service
In some cases generalize the AWS S3 functions for reuse

GitLab CI
---------
Add test case for api.sh tests with edge-commit and generic S3
This commit is contained in:
Ygal Blum 2022-05-24 09:25:29 +03:00
parent 335c597452
commit feb357e538
9 changed files with 325 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

75
internal/weldr/upload.go
View file

@ -40,12 +40,15 @@ type awsUploadSettings struct {
func (awsUploadSettings) isUploadSettings() {}
type awsS3UploadSettings struct {
Region string `json:"region"`
AccessKeyID string `json:"accessKeyID,omitempty"`
SecretAccessKey string `json:"secretAccessKey,omitempty"`
SessionToken string `json:"sessionToken,omitempty"`
Bucket string `json:"bucket"`
Key string `json:"key"`
Region string `json:"region"`
AccessKeyID string `json:"accessKeyID,omitempty"`
SecretAccessKey string `json:"secretAccessKey,omitempty"`
SessionToken string `json:"sessionToken,omitempty"`
Bucket string `json:"bucket"`
Key string `json:"key"`
Endpoint string `json:"endpoint"`
CABundle string `json:"ca_bundle"`
SkipSSLVerification bool `json:"skip_ssl_verification"`
}
func (awsS3UploadSettings) isUploadSettings() {}
@ -94,15 +97,6 @@ type ociUploadSettings struct {
func (ociUploadSettings) isUploadSettings() {}
type genericS3UploadSettings struct {
awsS3UploadSettings
Endpoint string `json:"endpoint"`
CABundle string `json:"ca_bundle"`
SkipSSLVerification bool `json:"skip_ssl_verification"`
}
func (genericS3UploadSettings) isUploadSettings() {}
type uploadRequest struct {
Provider string `json:"provider"`
ImageName string `json:"image_name"`
@ -137,7 +131,9 @@ func (u *uploadRequest) UnmarshalJSON(data []byte) error {
case "oci":
settings = new(ociUploadSettings)
case "generic.s3":
settings = new(genericS3UploadSettings)
// While the API still accepts provider type "generic.s3", the request is handled
// in the same way as for a request with provider type "aws.s3"
settings = new(awsS3UploadSettings)
default:
return errors.New("unexpected provider name")
}
@ -227,20 +223,6 @@ func targetsToUploadResponses(targets []*target.Target, state ComposeState) []up
// AccessKeyID and SecretAccessKey are intentionally not included.
}
uploads = append(uploads, upload)
case *target.GenericS3TargetOptions:
upload.ProviderName = "generic.s3"
upload.Settings = &genericS3UploadSettings{
awsS3UploadSettings: awsS3UploadSettings{
Region: options.Region,
Bucket: options.Bucket,
Key: options.Key,
// AccessKeyID and SecretAccessKey are intentionally not included.
},
Endpoint: options.Endpoint,
CABundle: options.CABundle,
SkipSSLVerification: options.SkipSSLVerification,
}
uploads = append(uploads, upload)
}
}
@ -270,13 +252,16 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target.
case *awsS3UploadSettings:
t.Name = "org.osbuild.aws.s3"
t.Options = &target.AWSS3TargetOptions{
Filename: imageType.Filename(),
Region: options.Region,
AccessKeyID: options.AccessKeyID,
SecretAccessKey: options.SecretAccessKey,
SessionToken: options.SessionToken,
Bucket: options.Bucket,
Key: options.Key,
Filename: imageType.Filename(),
Region: options.Region,
AccessKeyID: options.AccessKeyID,
SecretAccessKey: options.SecretAccessKey,
SessionToken: options.SessionToken,
Bucket: options.Bucket,
Key: options.Key,
Endpoint: options.Endpoint,
CABundle: options.CABundle,
SkipSSLVerification: options.SkipSSLVerification,
}
case *azureUploadSettings:
t.Name = "org.osbuild.azure"
@ -337,22 +322,6 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target.
Namespace: options.Namespace,
Compartment: options.Compartment,
}
case *genericS3UploadSettings:
t.Name = "org.osbuild.generic.s3"
t.Options = &target.GenericS3TargetOptions{
AWSS3TargetOptions: target.AWSS3TargetOptions{
Filename: imageType.Filename(),
Region: options.Region,
AccessKeyID: options.AccessKeyID,
SecretAccessKey: options.SecretAccessKey,
SessionToken: options.SessionToken,
Bucket: options.Bucket,
Key: options.Key,
},
Endpoint: options.Endpoint,
CABundle: options.CABundle,
SkipSSLVerification: options.SkipSSLVerification,
}
}
return &t