cloud/awscloud: switch s3 to v2 sdk

internal/cloud/awscloud/awscloud.go

@@ -1,29 +1,42 @@
 package awscloud
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"net/http"
 	"os"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/config"
+	credentialsv2 "github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/exp/slices"
 )
 
 type AWS struct {
-	uploader    *s3manager.Uploader
 	ec2         *ec2.EC2
 	ec2metadata *ec2metadata.EC2Metadata
-	s3          *s3.S3
+	s3          S3
+	s3uploader  S3Manager
+	s3presign   S3Presign
+}
+
+func newForTest(s3cli S3, upldr S3Manager, sign S3Presign) *AWS {
+	return &AWS{
+		s3:         s3cli,
+		s3uploader: upldr,
+		s3presign:  sign,
+	}
 }
 
 // Create a new session from the credentials and the region and returns an *AWS object initialized with it.
@@ -37,11 +50,27 @@ func newAwsFromCreds(creds *credentials.Credentials, region string) (*AWS, error
 		return nil, err
 	}
 
+	credsValue, err := creds.Get()
+	if err != nil {
+		return nil, err
+	}
+	cfg, err := config.LoadDefaultConfig(
+		context.Background(),
+		config.WithRegion(region),
+		config.WithCredentialsProvider(credentialsv2.NewStaticCredentialsProvider(
+			credsValue.AccessKeyID,
+			credsValue.SecretAccessKey,
+			credsValue.SessionToken,
+		)),
+	)
+
+	s3cli := s3.NewFromConfig(cfg)
 	return &AWS{
-		uploader:    s3manager.NewUploader(sess),
 		ec2:         ec2.New(sess),
 		ec2metadata: ec2metadata.New(sess),
-		s3:          s3.New(sess),
+		s3:          s3cli,
+		s3uploader:  manager.NewUploader(s3cli),
+		s3presign:   s3.NewPresignClient(s3cli),
 	}, nil
 }
 
@@ -94,6 +123,19 @@ func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoin
 		},
 	}
 
+	credsValue, err := creds.Get()
+	if err != nil {
+		return nil, err
+	}
+	v2OptionFuncs := []func(*config.LoadOptions) error{
+		config.WithRegion(region),
+		config.WithCredentialsProvider(credentialsv2.NewStaticCredentialsProvider(
+			credsValue.AccessKeyID,
+			credsValue.SecretAccessKey,
+			credsValue.SessionToken,
+		)),
+	}
+
 	if caBundle != "" {
 		caBundleReader, err := os.Open(caBundle)
 		if err != nil {
@@ -101,6 +143,7 @@ func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoin
 		}
 		defer caBundleReader.Close()
 		sessionOptions.CustomCABundle = caBundleReader
+		v2OptionFuncs = append(v2OptionFuncs, config.WithCustomCABundle(caBundleReader))
 	}
 
 	if skipSSLVerification {
@@ -109,6 +152,9 @@ func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoin
 		sessionOptions.Config.HTTPClient = &http.Client{
 			Transport: transport,
 		}
+		v2OptionFuncs = append(v2OptionFuncs, config.WithHTTPClient(&http.Client{
+			Transport: transport,
+		}))
 	}
 
 	sess, err := session.NewSessionWithOptions(sessionOptions)
@@ -116,11 +162,22 @@ func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoin
 		return nil, err
 	}
 
+	cfg, err := config.LoadDefaultConfig(
+		context.Background(),
+		v2OptionFuncs...,
+	)
+
+	s3cli := s3.NewFromConfig(cfg, func(options *s3.Options) {
+		options.BaseEndpoint = aws.String(endpoint)
+		options.UsePathStyle = true
+	})
+
 	return &AWS{
-		uploader:    s3manager.NewUploader(sess),
 		ec2:         ec2.New(sess),
 		ec2metadata: ec2metadata.New(sess),
-		s3:          s3.New(sess),
+		s3:          s3cli,
+		s3uploader:  manager.NewUploader(s3cli),
+		s3presign:   s3.NewPresignClient(s3cli),
 	}, nil
 }
 
@@ -142,7 +199,7 @@ func NewForEndpointFromFile(filename, endpoint, region, caBundle string, skipSSL
 	return newAwsFromCredsWithEndpoint(credentials.NewSharedCredentials(filename, "default"), region, endpoint, caBundle, skipSSLVerification)
 }
 
-func (a *AWS) Upload(filename, bucket, key string) (*s3manager.UploadOutput, error) {
+func (a *AWS) Upload(filename, bucket, key string) (*manager.UploadOutput, error) {
 	file, err := os.Open(filename)
 	if err != nil {
 		return nil, err
@@ -156,8 +213,9 @@ func (a *AWS) Upload(filename, bucket, key string) (*s3manager.UploadOutput, err
 	}()
 
 	logrus.Infof("[AWS] 🚀 Uploading image to S3: %s/%s", bucket, key)
-	return a.uploader.Upload(
+	return a.s3uploader.Upload(
-		&s3manager.UploadInput{
+		context.Background(),
+		&s3.PutObjectInput{
 			Bucket: aws.String(bucket),
 			Key:    aws.String(key),
 			Body:   file,
@@ -278,10 +336,13 @@ func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch str
 
 	// we no longer need the object in s3, let's just delete it
 	logrus.Infof("[AWS] 🧹 Deleting image from S3: %s/%s", bucket, key)
-	_, err = a.s3.DeleteObject(&s3.DeleteObjectInput{
+	_, err = a.s3.DeleteObject(
-		Bucket: aws.String(bucket),
+		context.Background(),
-		Key:    aws.String(key),
+		&s3.DeleteObjectInput{
-	})
+			Bucket: aws.String(bucket),
+			Key:    aws.String(key),
+		},
+	)
 	if err != nil {
 		return nil, err
 	}
@@ -597,30 +658,39 @@ func (a *AWS) DescribeImagesByTag(tagKey, tagValue string) ([]*ec2.Image, error)
 
 func (a *AWS) S3ObjectPresignedURL(bucket, objectKey string) (string, error) {
 	logrus.Infof("[AWS] 📋 Generating Presigned URL for S3 object %s/%s", bucket, objectKey)
-	req, _ := a.s3.GetObjectRequest(&s3.GetObjectInput{
+
-		Bucket: aws.String(bucket),
+	req, err := a.s3presign.PresignGetObject(
-		Key:    aws.String(objectKey),
+		context.Background(),
-	})
+		&s3.GetObjectInput{
-	url, err := req.Presign(7 * 24 * time.Hour) // maximum allowed
+			Bucket: aws.String(bucket),
+			Key:    aws.String(objectKey),
+		},
+		func(opts *s3.PresignOptions) {
+			opts.Expires = time.Duration(7 * 24 * time.Hour)
+		},
+	)
 	if err != nil {
 		return "", err
 	}
+
 	logrus.Info("[AWS] 🎉 S3 Presigned URL ready")
-	return url, nil
+	return req.URL, nil
 }
 
 func (a *AWS) MarkS3ObjectAsPublic(bucket, objectKey string) error {
 	logrus.Infof("[AWS] 👐 Making S3 object public %s/%s", bucket, objectKey)
-	_, err := a.s3.PutObjectAcl(&s3.PutObjectAclInput{
+	_, err := a.s3.PutObjectAcl(
-		Bucket: aws.String(bucket),
+		context.Background(),
-		Key:    aws.String(objectKey),
+		&s3.PutObjectAclInput{
-		ACL:    aws.String(s3.BucketCannedACLPublicRead),
+			Bucket: aws.String(bucket),
-	})
+			Key:    aws.String(objectKey),
+			ACL:    s3types.ObjectCannedACL(s3types.ObjectCannedACLPublicRead),
+		},
+	)
 	if err != nil {
 		return err
 	}
 	logrus.Info("[AWS] ✔️ Making S3 object public successful")
-
 	return nil
 }
 

internal/cloud/awscloud/awscloud_test.go

@@ -0,0 +1,35 @@
+package awscloud_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/osbuild/osbuild-composer/internal/cloud/awscloud"
+)
+
+func TestS3MarkObjectAsPublic(t *testing.T) {
+	aws := awscloud.NewForTest(&s3mock{t, "bucket", "object-key"}, nil, nil)
+	require.NotNil(t, aws)
+	require.NoError(t, aws.MarkS3ObjectAsPublic("bucket", "object-key"))
+}
+
+func TestS3Upload(t *testing.T) {
+	tmpDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(tmpDir, "file"), []byte("imanimage"), 0600))
+
+	aws := awscloud.NewForTest(nil, &s3upldrmock{t, "imanimage", "bucket", "object-key"}, nil)
+	require.NotNil(t, aws)
+	_, err := aws.Upload(filepath.Join(tmpDir, "file"), "bucket", "object-key")
+	require.NoError(t, err)
+}
+
+func TestS3ObjectPresignedURL(t *testing.T) {
+	aws := awscloud.NewForTest(nil, nil, &s3signmock{t, "bucket", "object-key"})
+	require.NotNil(t, aws)
+	url, err := aws.S3ObjectPresignedURL("bucket", "object-key")
+	require.NoError(t, err)
+	require.Equal(t, "https://url.real", url)
+}

internal/cloud/awscloud/client-interfaces.go

@@ -0,0 +1,22 @@
+package awscloud
+
+import (
+	"context"
+
+	"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+type S3 interface {
+	DeleteObject(context.Context, *s3.DeleteObjectInput, ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
+	PutObjectAcl(context.Context, *s3.PutObjectAclInput, ...func(*s3.Options)) (*s3.PutObjectAclOutput, error)
+}
+
+type S3Manager interface {
+	Upload(context.Context, *s3.PutObjectInput, ...func(*manager.Uploader)) (*manager.UploadOutput, error)
+}
+
+type S3Presign interface {
+	PresignGetObject(context.Context, *s3.GetObjectInput, ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error)
+}

internal/cloud/awscloud/export_test.go

@@ -0,0 +1,3 @@
+package awscloud
+
+var NewForTest = newForTest

internal/cloud/awscloud/mocks_test.go

@@ -0,0 +1,71 @@
+package awscloud_test
+
+import (
+	"context"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/stretchr/testify/require"
+)
+
+type s3mock struct {
+	t *testing.T
+
+	bucket string
+	key    string
+}
+
+func (m *s3mock) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput, optfns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
+	return nil, nil
+}
+
+func (m *s3mock) PutObjectAcl(ctx context.Context, input *s3.PutObjectAclInput, optfns ...func(*s3.Options)) (*s3.PutObjectAclOutput, error) {
+	require.Equal(m.t, m.bucket, *input.Bucket)
+	require.Equal(m.t, m.key, *input.Key)
+	require.Equal(m.t, s3types.ObjectCannedACL(s3types.ObjectCannedACLPublicRead), input.ACL)
+	return nil, nil
+}
+
+type s3upldrmock struct {
+	t *testing.T
+
+	contents string
+	bucket   string
+	key      string
+}
+
+func (m *s3upldrmock) Upload(ctx context.Context, input *s3.PutObjectInput, optfns ...func(*manager.Uploader)) (*manager.UploadOutput, error) {
+	body, err := io.ReadAll(input.Body)
+	require.NoError(m.t, err)
+	require.Equal(m.t, m.contents, string(body))
+	require.Equal(m.t, m.bucket, *input.Bucket)
+	require.Equal(m.t, m.key, *input.Key)
+	return nil, nil
+}
+
+type s3signmock struct {
+	t *testing.T
+
+	bucket string
+	key    string
+}
+
+func (m *s3signmock) PresignGetObject(ctx context.Context, input *s3.GetObjectInput, optfns ...func(*s3.PresignOptions)) (*v4.PresignedHTTPRequest, error) {
+	require.Equal(m.t, m.bucket, *input.Bucket)
+	require.Equal(m.t, m.key, *input.Key)
+
+	opts := &s3.PresignOptions{}
+	for _, fn := range optfns {
+		fn(opts)
+	}
+	require.Equal(m.t, time.Duration(7*24*time.Hour), opts.Expires)
+
+	return &v4.PresignedHTTPRequest{
+		URL: "https://url.real",
+	}, nil
+}