particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

OSBuild - add support for generic S3 services

Browse source 
jobimpl-osbuild
---------------
Add GenericS3Creds to struct
Add method to create AWS with Endpoint for Generic S3 (with its own credentials file)
Move uploading to S3 and result handling to a separate method (along with the special VMDK handling)
adjust the AWS S3 case to the new method
Implement a new case for uploading to a generic S3 service

awscloud
--------
Add wrapper methods for endpoint support
Set the endpoint to the AWS session
Set s3ForcePathStyle to true if endpoint was set

Target
------
Define a new target type for the GenericS3Target and Options
Handle unmarshaling of the target options and result for the Generic S3

Weldr
-----
Add support for only uploading to AWS S3
Define new structures for AWS S3 and Generic S3 (based on AWS S3)
Handle unmarshaling of the providers settings' upload settings

main
----
Add a section in the main config for the Generic S3 service for credentials
If provided pass the credentials file name to the osbuild job implementation

Upload Utility
--------------
Add upload-generic-s3 utility

Makefile
------
Do not fail if the bin directory already exists

Tests
-----
Add test cases for both AWS and a generic S3 server
Add a generic s3_test.sh file for both test cases and add it to the tests RPM spec
Adjust the libvirt test case script to support already created images
GitLabCI - Extend the libvirt test case to include the two new tests
This commit is contained in:
Ygal Blum 2022-03-28 15:34:30 +03:00 • committed by Tomáš Hozza
parent 01880a76a2
commit bee14bf392
15 changed files with 684 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

33
internal/cloud/awscloud/awscloud.go
View file

@ -22,11 +22,14 @@ type AWS struct {
}
// Create a new session from the credentials and the region and returns an *AWS object initialized with it.
func newAwsFromCreds(creds *credentials.Credentials, region string) (*AWS, error) {
func newAwsFromCreds(creds *credentials.Credentials, region string, endpoint *string) (*AWS, error) {
// Create a Session with a custom region
s3ForcePathStyle := endpoint != nil
sess, err := session.NewSession(&aws.Config{
Credentials: creds,
Region: aws.String(region),
Credentials: creds,
Region: aws.String(region),
Endpoint: endpoint,
S3ForcePathStyle: &s3ForcePathStyle,
})
if err != nil {
return nil, err
@ -41,7 +44,7 @@ func newAwsFromCreds(creds *credentials.Credentials, region string) (*AWS, error
// Initialize a new AWS object from individual bits. SessionToken is optional
func New(region string, accessKeyID string, accessKey string, sessionToken string) (*AWS, error) {
return newAwsFromCreds(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region)
return newAwsFromCreds(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region, nil)
}
// Initializes a new AWS object with the credentials info found at filename's location.
@ -54,13 +57,31 @@ func New(region string, accessKeyID string, accessKey string, sessionToken strin
// "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to
// $HOME/.aws/credentials.
func NewFromFile(filename string, region string) (*AWS, error) {
return newAwsFromCreds(credentials.NewSharedCredentials(filename, "default"), region)
return newAwsFromCreds(credentials.NewSharedCredentials(filename, "default"), region, nil)
}
// Initialize a new AWS object from defaults.
// Looks for env variables, shared credential file, and EC2 Instance Roles.
func NewDefault(region string) (*AWS, error) {
return newAwsFromCreds(nil, region)
return newAwsFromCreds(nil, region, nil)
}
// Initialize a new AWS object targeting a specific endpoint from individual bits. SessionToken is optional
func NewForEndpoint(endpoint, region string, accessKeyID string, accessKey string, sessionToken string) (*AWS, error) {
return newAwsFromCreds(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region, &endpoint)
}
// Initializes a new AWS object targeting a specific endpoint with the credentials info found at filename's location.
// The credential files should match the AWS format, such as:
// [default]
// aws_access_key_id = secretString1
// aws_secret_access_key = secretString2
//
// If filename is empty the underlying function will look for the
// "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to
// $HOME/.aws/credentials.
func NewForEndpointFromFile(filename string, endpoint, region string) (*AWS, error) {
return newAwsFromCreds(credentials.NewSharedCredentials(filename, "default"), region, &endpoint)
}
func (a *AWS) Upload(filename, bucket, key string) (*s3manager.UploadOutput, error) {