particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

cloud/awscloud: switch s3 to v2 sdk

Browse source
This commit is contained in:
Sanne Raymaekers 2024-08-01 13:02:59 +02:00
parent 5e3bc8a705
commit 8d158f6031
5 changed files with 228 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

124
internal/cloud/awscloud/awscloud.go
View file

@ -1,29 +1,42 @@
package awscloud
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"os"
"time"
"github.com/aws/aws-sdk-go-v2/config"
credentialsv2 "github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
)
type AWS struct {
uploader *s3manager.Uploader
ec2 *ec2.EC2
ec2metadata *ec2metadata.EC2Metadata
s3 *s3.S3
s3 S3
s3uploader S3Manager
s3presign S3Presign
}
func newForTest(s3cli S3, upldr S3Manager, sign S3Presign) *AWS {
return &AWS{
s3: s3cli,
s3uploader: upldr,
s3presign: sign,
}
}
// Create a new session from the credentials and the region and returns an *AWS object initialized with it.
@ -37,11 +50,27 @@ func newAwsFromCreds(creds *credentials.Credentials, region string) (*AWS, error
return nil, err
}
credsValue, err := creds.Get()
if err != nil {
return nil, err
}
cfg, err := config.LoadDefaultConfig(
context.Background(),
config.WithRegion(region),
config.WithCredentialsProvider(credentialsv2.NewStaticCredentialsProvider(
credsValue.AccessKeyID,
credsValue.SecretAccessKey,
credsValue.SessionToken,
)),
)
s3cli := s3.NewFromConfig(cfg)
return &AWS{
uploader: s3manager.NewUploader(sess),
ec2: ec2.New(sess),
ec2metadata: ec2metadata.New(sess),
s3: s3.New(sess),
s3: s3cli,
s3uploader: manager.NewUploader(s3cli),
s3presign: s3.NewPresignClient(s3cli),
}, nil
}
@ -94,6 +123,19 @@ func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoin
},
}
credsValue, err := creds.Get()
if err != nil {
return nil, err
}
v2OptionFuncs := []func(*config.LoadOptions) error{
config.WithRegion(region),
config.WithCredentialsProvider(credentialsv2.NewStaticCredentialsProvider(
credsValue.AccessKeyID,
credsValue.SecretAccessKey,
credsValue.SessionToken,
)),
}
if caBundle != "" {
caBundleReader, err := os.Open(caBundle)
if err != nil {
@ -101,6 +143,7 @@ func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoin
}
defer caBundleReader.Close()
sessionOptions.CustomCABundle = caBundleReader
v2OptionFuncs = append(v2OptionFuncs, config.WithCustomCABundle(caBundleReader))
}
if skipSSLVerification {
@ -109,6 +152,9 @@ func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoin
sessionOptions.Config.HTTPClient = &http.Client{
Transport: transport,
}
v2OptionFuncs = append(v2OptionFuncs, config.WithHTTPClient(&http.Client{
Transport: transport,
}))
}
sess, err := session.NewSessionWithOptions(sessionOptions)
@ -116,11 +162,22 @@ func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoin
return nil, err
}
cfg, err := config.LoadDefaultConfig(
context.Background(),
v2OptionFuncs...,
)
s3cli := s3.NewFromConfig(cfg, func(options *s3.Options) {
options.BaseEndpoint = aws.String(endpoint)
options.UsePathStyle = true
})
return &AWS{
uploader: s3manager.NewUploader(sess),
ec2: ec2.New(sess),
ec2metadata: ec2metadata.New(sess),
s3: s3.New(sess),
s3: s3cli,
s3uploader: manager.NewUploader(s3cli),
s3presign: s3.NewPresignClient(s3cli),
}, nil
}
@ -142,7 +199,7 @@ func NewForEndpointFromFile(filename, endpoint, region, caBundle string, skipSSL
return newAwsFromCredsWithEndpoint(credentials.NewSharedCredentials(filename, "default"), region, endpoint, caBundle, skipSSLVerification)
}
func (a *AWS) Upload(filename, bucket, key string) (*s3manager.UploadOutput, error) {
func (a *AWS) Upload(filename, bucket, key string) (*manager.UploadOutput, error) {
file, err := os.Open(filename)
if err != nil {
return nil, err
@ -156,8 +213,9 @@ func (a *AWS) Upload(filename, bucket, key string) (*s3manager.UploadOutput, err
}()
logrus.Infof("[AWS] 🚀 Uploading image to S3: %s/%s", bucket, key)
return a.uploader.Upload(
&s3manager.UploadInput{
return a.s3uploader.Upload(
context.Background(),
&s3.PutObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
Body: file,
@ -278,10 +336,13 @@ func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch str
// we no longer need the object in s3, let's just delete it
logrus.Infof("[AWS] 🧹 Deleting image from S3: %s/%s", bucket, key)
_, err = a.s3.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
})
_, err = a.s3.DeleteObject(
context.Background(),
&s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(key),
},
)
if err != nil {
return nil, err
}
@ -597,30 +658,39 @@ func (a *AWS) DescribeImagesByTag(tagKey, tagValue string) ([]*ec2.Image, error)
func (a *AWS) S3ObjectPresignedURL(bucket, objectKey string) (string, error) {
logrus.Infof("[AWS] 📋 Generating Presigned URL for S3 object %s/%s", bucket, objectKey)
req, _ := a.s3.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectKey),
})
url, err := req.Presign(7 * 24 * time.Hour) // maximum allowed
req, err := a.s3presign.PresignGetObject(
context.Background(),
&s3.GetObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectKey),
},
func(opts *s3.PresignOptions) {
opts.Expires = time.Duration(7 * 24 * time.Hour)
},
)
if err != nil {
return "", err
}
logrus.Info("[AWS] 🎉 S3 Presigned URL ready")
return url, nil
return req.URL, nil
}
func (a *AWS) MarkS3ObjectAsPublic(bucket, objectKey string) error {
logrus.Infof("[AWS] 👐 Making S3 object public %s/%s", bucket, objectKey)
_, err := a.s3.PutObjectAcl(&s3.PutObjectAclInput{
Bucket: aws.String(bucket),
Key: aws.String(objectKey),
ACL: aws.String(s3.BucketCannedACLPublicRead),
})
_, err := a.s3.PutObjectAcl(
context.Background(),
&s3.PutObjectAclInput{
Bucket: aws.String(bucket),
Key: aws.String(objectKey),
ACL: s3types.ObjectCannedACL(s3types.ObjectCannedACLPublicRead),
},
)
if err != nil {
return err
}
logrus.Info("[AWS] ✔️ Making S3 object public successful")
return nil
}

35
internal/cloud/awscloud/awscloud_test.go Normal file
View file

@ -0,0 +1,35 @@
package awscloud_test
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/require"
"github.com/osbuild/osbuild-composer/internal/cloud/awscloud"
)
func TestS3MarkObjectAsPublic(t *testing.T) {
aws := awscloud.NewForTest(&s3mock{t, "bucket", "object-key"}, nil, nil)
require.NotNil(t, aws)
require.NoError(t, aws.MarkS3ObjectAsPublic("bucket", "object-key"))
}
func TestS3Upload(t *testing.T) {
tmpDir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "file"), []byte("imanimage"), 0600))
aws := awscloud.NewForTest(nil, &s3upldrmock{t, "imanimage", "bucket", "object-key"}, nil)
require.NotNil(t, aws)
_, err := aws.Upload(filepath.Join(tmpDir, "file"), "bucket", "object-key")
require.NoError(t, err)
}
func TestS3ObjectPresignedURL(t *testing.T) {
aws := awscloud.NewForTest(nil, nil, &s3signmock{t, "bucket", "object-key"})
require.NotNil(t, aws)
url, err := aws.S3ObjectPresignedURL("bucket", "object-key")
require.NoError(t, err)
require.Equal(t, "https://url.real", url)
}

22
internal/cloud/awscloud/client-interfaces.go Normal file
View file

@ -0,0 +1,22 @@
package awscloud
import (
"context"
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
type S3 interface {
DeleteObject(context.Context, *s3.DeleteObjectInput, ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
PutObjectAcl(context.Context, *s3.PutObjectAclInput, ...func(*s3.Options)) (*s3.PutObjectAclOutput, error)
}
type S3Manager interface {
Upload(context.Context, *s3.PutObjectInput, ...func(*manager.Uploader)) (*manager.UploadOutput, error)
}
type S3Presign interface {
PresignGetObject(context.Context, *s3.GetObjectInput, ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error)
}

3
internal/cloud/awscloud/export_test.go Normal file
View file

@ -0,0 +1,3 @@
package awscloud
var NewForTest = newForTest

71
internal/cloud/awscloud/mocks_test.go Normal file
View file

@ -0,0 +1,71 @@
package awscloud_test
import (
"context"
"io"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/stretchr/testify/require"
)
type s3mock struct {
t *testing.T
bucket string
key string
}
func (m *s3mock) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput, optfns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
return nil, nil
}
func (m *s3mock) PutObjectAcl(ctx context.Context, input *s3.PutObjectAclInput, optfns ...func(*s3.Options)) (*s3.PutObjectAclOutput, error) {
require.Equal(m.t, m.bucket, *input.Bucket)
require.Equal(m.t, m.key, *input.Key)
require.Equal(m.t, s3types.ObjectCannedACL(s3types.ObjectCannedACLPublicRead), input.ACL)
return nil, nil
}
type s3upldrmock struct {
t *testing.T
contents string
bucket string
key string
}
func (m *s3upldrmock) Upload(ctx context.Context, input *s3.PutObjectInput, optfns ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
body, err := io.ReadAll(input.Body)
require.NoError(m.t, err)
require.Equal(m.t, m.contents, string(body))
require.Equal(m.t, m.bucket, *input.Bucket)
require.Equal(m.t, m.key, *input.Key)
return nil, nil
}
type s3signmock struct {
t *testing.T
bucket string
key string
}
func (m *s3signmock) PresignGetObject(ctx context.Context, input *s3.GetObjectInput, optfns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) {
require.Equal(m.t, m.bucket, *input.Bucket)
require.Equal(m.t, m.key, *input.Key)
opts := &s3.PresignOptions{}
for _, fn := range optfns {
fn(opts)
}
require.Equal(m.t, time.Duration(7*24*time.Hour), opts.Expires)
return &v4.PresignedHTTPRequest{
URL: "https://url.real",
}, nil
}