particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

OSBuild - add support for generic S3 services

Browse source 
jobimpl-osbuild
---------------
Add GenericS3Creds to struct
Add method to create AWS with Endpoint for Generic S3 (with its own credentials file)
Move uploading to S3 and result handling to a separate method (along with the special VMDK handling)
adjust the AWS S3 case to the new method
Implement a new case for uploading to a generic S3 service

awscloud
--------
Add wrapper methods for endpoint support
Set the endpoint to the AWS session
Set s3ForcePathStyle to true if endpoint was set

Target
------
Define a new target type for the GenericS3Target and Options
Handle unmarshaling of the target options and result for the Generic S3

Weldr
-----
Add support for only uploading to AWS S3
Define new structures for AWS S3 and Generic S3 (based on AWS S3)
Handle unmarshaling of the providers settings' upload settings

main
----
Add a section in the main config for the Generic S3 service for credentials
If provided pass the credentials file name to the osbuild job implementation

Upload Utility
--------------
Add upload-generic-s3 utility

Makefile
------
Do not fail if the bin directory already exists

Tests
-----
Add test cases for both AWS and a generic S3 server
Add a generic s3_test.sh file for both test cases and add it to the tests RPM spec
Adjust the libvirt test case script to support already created images
GitLabCI - Extend the libvirt test case to include the two new tests
This commit is contained in:
Ygal Blum 2022-03-28 15:34:30 +03:00 • committed by Tomáš Hozza
parent 01880a76a2
commit bee14bf392
15 changed files with 684 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

68
internal/weldr/upload.go
View file

@ -36,6 +36,17 @@ type awsUploadSettings struct {
func (awsUploadSettings) isUploadSettings() {}
type awsS3UploadSettings struct {
Region string `json:"region"`
AccessKeyID string `json:"accessKeyID,omitempty"`
SecretAccessKey string `json:"secretAccessKey,omitempty"`
SessionToken string `json:"sessionToken,omitempty"`
Bucket string `json:"bucket"`
Key string `json:"key"`
}
func (awsS3UploadSettings) isUploadSettings() {}
type azureUploadSettings struct {
StorageAccount string `json:"storageAccount,omitempty"`
StorageAccessKey string `json:"storageAccessKey,omitempty"`
@ -68,6 +79,13 @@ type ociUploadSettings struct {
func (ociUploadSettings) isUploadSettings() {}
type genericS3UploadSettings struct {
awsS3UploadSettings
Endpoint string `json:"endpoint"`
}
func (genericS3UploadSettings) isUploadSettings() {}
type uploadRequest struct {
Provider string `json:"provider"`
ImageName string `json:"image_name"`
@ -93,10 +111,14 @@ func (u *uploadRequest) UnmarshalJSON(data []byte) error {
settings = new(azureUploadSettings)
case "aws":
settings = new(awsUploadSettings)
case "aws.s3":
settings = new(awsS3UploadSettings)
case "vmware":
settings = new(vmwareUploadSettings)
case "oci":
settings = new(ociUploadSettings)
case "generic.s3":
settings = new(genericS3UploadSettings)
default:
return errors.New("unexpected provider name")
}
@ -167,6 +189,27 @@ func targetsToUploadResponses(targets []*target.Target, state ComposeState) []up
// Username and Password are intentionally not included.
}
uploads = append(uploads, upload)
case *target.AWSS3TargetOptions:
upload.ProviderName = "aws.s3"
upload.Settings = &awsS3UploadSettings{
Region: options.Region,
Bucket: options.Bucket,
Key: options.Key,
// AccessKeyID and SecretAccessKey are intentionally not included.
}
uploads = append(uploads, upload)
case *target.GenericS3TargetOptions:
upload.ProviderName = "generic.s3"
upload.Settings = &genericS3UploadSettings{
awsS3UploadSettings: awsS3UploadSettings{
Region: options.Region,
Bucket: options.Bucket,
Key: options.Key,
// AccessKeyID and SecretAccessKey are intentionally not included.
},
Endpoint: options.Endpoint,
}
uploads = append(uploads, upload)
}
}
@ -193,6 +236,17 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target.
Bucket: options.Bucket,
Key: options.Key,
}
case *awsS3UploadSettings:
t.Name = "org.osbuild.aws.s3"
t.Options = &target.AWSS3TargetOptions{
Filename: imageType.Filename(),
Region: options.Region,
AccessKeyID: options.AccessKeyID,
SecretAccessKey: options.SecretAccessKey,
SessionToken: options.SessionToken,
Bucket: options.Bucket,
Key: options.Key,
}
case *azureUploadSettings:
t.Name = "org.osbuild.azure"
t.Options = &target.AzureTargetOptions{
@ -225,6 +279,20 @@ func uploadRequestToTarget(u uploadRequest, imageType distro.ImageType) *target.
Namespace: options.Namespace,
Compartment: options.Compartment,
}
case *genericS3UploadSettings:
t.Name = "org.osbuild.generic.s3"
t.Options = &target.GenericS3TargetOptions{
AWSS3TargetOptions: target.AWSS3TargetOptions{
Filename: imageType.Filename(),
Region: options.Region,
AccessKeyID: options.AccessKeyID,
SecretAccessKey: options.SecretAccessKey,
SessionToken: options.SessionToken,
Bucket: options.Bucket,
Key: options.Key,
},
Endpoint: options.Endpoint,
}
}
return &t