diff --git a/internal/cloud/awscloud/awscloud.go b/internal/cloud/awscloud/awscloud.go index dbf4c952d..d19176a81 100644 --- a/internal/cloud/awscloud/awscloud.go +++ b/internal/cloud/awscloud/awscloud.go @@ -18,10 +18,15 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + images_awscloud "github.com/osbuild/images/pkg/cloud/awscloud" "github.com/sirupsen/logrus" ) type AWS struct { + // awscloud.AWS from the osbuild/images package implements all of the methods + // related to image upload and sharing. + *images_awscloud.AWS + ec2 EC2 ec2imds EC2Imds s3 S3 @@ -43,9 +48,10 @@ func newForTest(ec2cli EC2, ec2imds EC2Imds, s3cli S3, upldr S3Manager, sign S3P // Create a new session from the credentials and the region and returns an *AWS object initialized with it. // /creds credentials.StaticCredentialsProvider, region string -func newAwsFromConfig(cfg aws.Config) *AWS { +func newAwsFromConfig(cfg aws.Config, imagesAWS *images_awscloud.AWS) *AWS { s3cli := s3.NewFromConfig(cfg) return &AWS{ + AWS: imagesAWS, ec2: ec2.NewFromConfig(cfg), ec2imds: imds.NewFromConfig(cfg), s3: s3cli, @@ -65,7 +71,13 @@ func New(region string, accessKeyID string, accessKey string, sessionToken strin if err != nil { return nil, err } - aws := newAwsFromConfig(cfg) + + imagesAWS, err := images_awscloud.New(region, accessKeyID, accessKey, sessionToken) + if err != nil { + return nil, fmt.Errorf("failed to create images AWS client: %w", err) + } + + aws := newAwsFromConfig(cfg, imagesAWS) return aws, nil } @@ -90,7 +102,13 @@ func NewFromFile(filename string, region string) (*AWS, error) { if err != nil { return nil, err } - aws := newAwsFromConfig(cfg) + + imagesAWS, err := images_awscloud.NewFromFile(filename, region) + if err != nil { + return nil, fmt.Errorf("failed to create images AWS client: %w", err) + } + + aws := newAwsFromConfig(cfg, imagesAWS) return aws, nil } @@ -104,7 +122,13 @@ func NewDefault(region string) (*AWS, error) { if err != nil { return nil, err } - aws := newAwsFromConfig(cfg) + + imagesAWS, err := images_awscloud.NewDefault(region) + if err != nil { + return nil, fmt.Errorf("failed to create images AWS client: %w", err) + } + + aws := newAwsFromConfig(cfg, imagesAWS) return aws, nil } @@ -120,7 +144,7 @@ func RegionFromInstanceMetadata() (string, error) { } // Create a new session from the credentials and the region and returns an *AWS object initialized with it. -func newAwsFromCredsWithEndpoint(creds config.LoadOptionsFunc, region, endpoint, caBundle string, skipSSLVerification bool) (*AWS, error) { +func newAwsFromCredsWithEndpoint(creds config.LoadOptionsFunc, region, endpoint, caBundle string, skipSSLVerification bool, imagesAWS *images_awscloud.AWS) (*AWS, error) { // Create a Session with a custom region v2OptionFuncs := []func(*config.LoadOptions) error{ config.WithRegion(region), @@ -158,6 +182,7 @@ func newAwsFromCredsWithEndpoint(creds config.LoadOptionsFunc, region, endpoint, }) return &AWS{ + AWS: imagesAWS, ec2: ec2.NewFromConfig(cfg), ec2imds: imds.NewFromConfig(cfg), s3: s3cli, @@ -169,7 +194,11 @@ func newAwsFromCredsWithEndpoint(creds config.LoadOptionsFunc, region, endpoint, // Initialize a new AWS object targeting a specific endpoint from individual bits. SessionToken is optional func NewForEndpoint(endpoint, region, accessKeyID, accessKey, sessionToken, caBundle string, skipSSLVerification bool) (*AWS, error) { - return newAwsFromCredsWithEndpoint(config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKey, sessionToken)), region, endpoint, caBundle, skipSSLVerification) + imagesAWS, err := images_awscloud.NewForEndpoint(endpoint, region, accessKeyID, accessKey, sessionToken, caBundle, skipSSLVerification) + if err != nil { + return nil, fmt.Errorf("failed to create images AWS client: %w", err) + } + return newAwsFromCredsWithEndpoint(config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKey, sessionToken)), region, endpoint, caBundle, skipSSLVerification, imagesAWS) } // Initializes a new AWS object targeting a specific endpoint with the credentials info found at filename's location. @@ -182,7 +211,11 @@ func NewForEndpoint(endpoint, region, accessKeyID, accessKey, sessionToken, caBu // "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to // $HOME/.aws/credentials. func NewForEndpointFromFile(filename, endpoint, region, caBundle string, skipSSLVerification bool) (*AWS, error) { - return newAwsFromCredsWithEndpoint(config.WithSharedCredentialsFiles([]string{filename, "default"}), region, endpoint, caBundle, skipSSLVerification) + imagesAWS, err := images_awscloud.NewForEndpointFromFile(filename, endpoint, region, caBundle, skipSSLVerification) + if err != nil { + return nil, fmt.Errorf("failed to create images AWS client: %w", err) + } + return newAwsFromCredsWithEndpoint(config.WithSharedCredentialsFiles([]string{filename, "default"}), region, endpoint, caBundle, skipSSLVerification, imagesAWS) } // This is used by the internal/boot test, which access the ec2 apis directly diff --git a/vendor/github.com/osbuild/images/pkg/cloud/awscloud/awscloud.go b/vendor/github.com/osbuild/images/pkg/cloud/awscloud/awscloud.go new file mode 100644 index 000000000..13d457844 --- /dev/null +++ b/vendor/github.com/osbuild/images/pkg/cloud/awscloud/awscloud.go @@ -0,0 +1,734 @@ +package awscloud + +import ( + "context" + "crypto/tls" + "encoding/base64" + "fmt" + "io" + "net/http" + "os" + "time" + + "slices" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/s3" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/osbuild/images/pkg/arch" + "github.com/osbuild/images/pkg/olog" + "github.com/osbuild/images/pkg/platform" +) + +type AWS struct { + ec2 ec2Client + s3 s3Client + s3uploader s3Uploader + s3presign s3Presign +} + +// Allow to mock the EC2 SnapshotImportedWaiter for testing purposes +var newSnapshotImportedWaiterEC2 = func(client ec2.DescribeImportSnapshotTasksAPIClient, optFns ...func(*ec2.SnapshotImportedWaiterOptions)) snapshotImportedWaiterEC2 { + return ec2.NewSnapshotImportedWaiter(client, optFns...) +} + +// Allow to mock the EC2 NewInstanceRunningWaiter for testing purposes +var newInstanceRunningWaiterEC2 = func(client ec2.DescribeInstancesAPIClient, optFns ...func(*ec2.InstanceRunningWaiterOptions)) instanceRunningWaiterEC2 { + return ec2.NewInstanceRunningWaiter(client, optFns...) +} + +var newTerminateInstancesWaiterEC2 = func(client ec2.DescribeInstancesAPIClient, optFns ...func(*ec2.InstanceTerminatedWaiterOptions)) instanceTerminatedWaiterEC2 { + return ec2.NewInstanceTerminatedWaiter(client, optFns...) +} + +// S3PermissionsMatrix Maps a requested permission to all permissions that are sufficient for the requested one +var S3PermissionsMatrix = map[s3types.Permission][]s3types.Permission{ + s3types.PermissionRead: {s3types.PermissionRead, s3types.PermissionWrite, s3types.PermissionFullControl}, + s3types.PermissionWrite: {s3types.PermissionWrite, s3types.PermissionFullControl}, + s3types.PermissionFullControl: {s3types.PermissionFullControl}, + s3types.PermissionReadAcp: {s3types.PermissionReadAcp, s3types.PermissionWriteAcp}, + s3types.PermissionWriteAcp: {s3types.PermissionWriteAcp}, +} + +// Create a new session from the credentials and the region and returns an *AWS object initialized with it. +func newAwsFromConfig(cfg aws.Config) *AWS { + s3cli := s3.NewFromConfig(cfg) + return &AWS{ + ec2: ec2.NewFromConfig(cfg), + s3: s3cli, + s3uploader: s3manager.NewUploader(s3cli), + s3presign: s3.NewPresignClient(s3cli), + } +} + +// Initialize a new AWS object from individual bits. SessionToken is optional +func New(region string, accessKeyID string, accessKey string, sessionToken string) (*AWS, error) { + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(region), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKey, sessionToken)), + ) + if err != nil { + return nil, err + } + aws := newAwsFromConfig(cfg) + return aws, nil +} + +// Initializes a new AWS object with the credentials info found at filename's location. +// The credential files should match the AWS format, such as: +// [default] +// aws_access_key_id = secretString1 +// aws_secret_access_key = secretString2 +// +// If filename is empty the underlying function will look for the +// "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to +// $HOME/.aws/credentials. +func NewFromFile(filename string, region string) (*AWS, error) { + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(region), + config.WithSharedCredentialsFiles([]string{ + filename, + "default", + }), + ) + if err != nil { + return nil, err + } + aws := newAwsFromConfig(cfg) + return aws, nil +} + +// Initialize a new AWS object from defaults. +// Looks for env variables, shared credential file, and EC2 Instance Roles. +func NewDefault(region string) (*AWS, error) { + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(region), + ) + if err != nil { + return nil, err + } + aws := newAwsFromConfig(cfg) + return aws, nil +} + +// Create a new session from the credentials and the region and returns an *AWS object initialized with it. +func newAwsFromCredsWithEndpoint(optsFunc config.LoadOptionsFunc, region, endpoint, caBundle string, skipSSLVerification bool) (*AWS, error) { + // Create a Session with a custom region + optionFuncs := []func(*config.LoadOptions) error{ + config.WithRegion(region), + optsFunc, + } + + if caBundle != "" { + caBundleReader, err := os.Open(caBundle) + if err != nil { + return nil, err + } + defer caBundleReader.Close() + optionFuncs = append(optionFuncs, config.WithCustomCABundle(caBundleReader)) + } + + if skipSSLVerification { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402 + optionFuncs = append(optionFuncs, config.WithHTTPClient(&http.Client{ + Transport: transport, + })) + } + + cfg, err := config.LoadDefaultConfig( + context.TODO(), + optionFuncs..., + ) + if err != nil { + return nil, err + } + + s3cli := s3.NewFromConfig(cfg, func(options *s3.Options) { + options.BaseEndpoint = aws.String(endpoint) + options.UsePathStyle = true + }) + + return &AWS{ + ec2: ec2.NewFromConfig(cfg), + s3: s3cli, + s3uploader: s3manager.NewUploader(s3cli), + s3presign: s3.NewPresignClient(s3cli), + }, nil +} + +// Initialize a new AWS object targeting a specific endpoint from individual bits. SessionToken is optional +func NewForEndpoint(endpoint, region, accessKeyID, accessKey, sessionToken, caBundle string, skipSSLVerification bool) (*AWS, error) { + return newAwsFromCredsWithEndpoint(config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKey, sessionToken)), region, endpoint, caBundle, skipSSLVerification) +} + +// Initializes a new AWS object targeting a specific endpoint with the credentials info found at filename's location. +// The credential files should match the AWS format, such as: +// [default] +// aws_access_key_id = secretString1 +// aws_secret_access_key = secretString2 +// +// If filename is empty the underlying function will look for the +// "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to +// $HOME/.aws/credentials. +func NewForEndpointFromFile(filename, endpoint, region, caBundle string, skipSSLVerification bool) (*AWS, error) { + return newAwsFromCredsWithEndpoint(config.WithSharedCredentialsFiles([]string{filename, "default"}), region, endpoint, caBundle, skipSSLVerification) +} + +func (a *AWS) Upload(filename, bucket, key string) (*s3manager.UploadOutput, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + + defer func() { + err := file.Close() + if err != nil { + olog.Printf("[AWS] โ€ผ Failed to close the file uploaded to S3๏ธ: %v", err) + } + }() + return a.UploadFromReader(file, bucket, key) +} + +func (a *AWS) UploadFromReader(r io.Reader, bucket, key string) (*s3manager.UploadOutput, error) { + olog.Printf("[AWS] ๐Ÿš€ Uploading image to S3: %s/%s", bucket, key) + return a.s3uploader.Upload( + context.TODO(), + &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: r, + }, + ) +} + +func ec2BootMode(bootMode *platform.BootMode) (ec2types.BootModeValues, error) { + if bootMode == nil { + return ec2types.BootModeValues(""), nil + } + + switch *bootMode { + case platform.BOOT_LEGACY: + return ec2types.BootModeValuesLegacyBios, nil + case platform.BOOT_UEFI: + return ec2types.BootModeValuesUefi, nil + case platform.BOOT_HYBRID: + return ec2types.BootModeValuesUefiPreferred, nil + default: + return ec2types.BootModeValues(""), fmt.Errorf("invalid boot mode: %s", *bootMode) + } +} + +// Register is a function that imports a snapshot, waits for the snapshot to +// fully import, tags the snapshot, cleans up the image in S3, and registers +// an AMI in AWS. +// The caller can optionally specify the boot mode of the AMI. If the boot +// mode is not specified, then the instances launched from this AMI use the +// default boot mode value of the instance type. +func (a *AWS) Register(name, bucket, key string, shareWith []string, architecture arch.Arch, bootMode *platform.BootMode, importRole *string) (string, string, error) { + rpmArchToEC2Arch := map[arch.Arch]ec2types.ArchitectureValues{ + arch.ARCH_X86_64: ec2types.ArchitectureValuesX8664, + arch.ARCH_AARCH64: ec2types.ArchitectureValuesArm64, + } + + ec2Arch, validArch := rpmArchToEC2Arch[architecture] + if !validArch { + return "", "", fmt.Errorf("ec2 doesn't support the following arch: %s", architecture) + } + + ec2BootMode, err := ec2BootMode(bootMode) + if err != nil { + return "", "", fmt.Errorf("ec2 doesn't support the following boot mode: %s", bootMode) + } + + olog.Printf("[AWS] ๐Ÿ“ฅ Importing snapshot from image: %s/%s", bucket, key) + snapshotDescription := fmt.Sprintf("Image Builder AWS Import of %s", name) + importTaskOutput, err := a.ec2.ImportSnapshot( + context.TODO(), + &ec2.ImportSnapshotInput{ + Description: aws.String(snapshotDescription), + DiskContainer: &ec2types.SnapshotDiskContainer{ + UserBucket: &ec2types.UserBucket{ + S3Bucket: aws.String(bucket), + S3Key: aws.String(key), + }, + }, + RoleName: importRole, + }, + ) + if err != nil { + olog.Printf("[AWS] error importing snapshot: %s", err) + return "", "", err + } + + olog.Printf("[AWS] ๐Ÿšš Waiting for snapshot to finish importing: %s", *importTaskOutput.ImportTaskId) + snapWaiter := newSnapshotImportedWaiterEC2(a.ec2) + snapWaitOutput, err := snapWaiter.WaitForOutput( + context.TODO(), + &ec2.DescribeImportSnapshotTasksInput{ + ImportTaskIds: []string{ + *importTaskOutput.ImportTaskId, + }, + }, + time.Hour*24, + ) + if err != nil { + return "", "", err + } + + snapshotTaskStatus := *snapWaitOutput.ImportSnapshotTasks[0].SnapshotTaskDetail.Status + if snapshotTaskStatus != "completed" { + return "", "", fmt.Errorf("Unable to import snapshot, task result: %v, msg: %v", snapshotTaskStatus, *snapWaitOutput.ImportSnapshotTasks[0].SnapshotTaskDetail.StatusMessage) + } + + // we no longer need the object in s3, let's just delete it + olog.Printf("[AWS] ๐Ÿงน Deleting image from S3: %s/%s", bucket, key) + err = a.DeleteObject(bucket, key) + if err != nil { + return "", "", err + } + + snapshotID := *snapWaitOutput.ImportSnapshotTasks[0].SnapshotTaskDetail.SnapshotId + // Tag the snapshot with the image name. + _, err = a.ec2.CreateTags( + context.TODO(), + &ec2.CreateTagsInput{ + Resources: []string{snapshotID}, + Tags: []ec2types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(name), + }, + }, + }, + ) + if err != nil { + return "", "", err + } + + olog.Printf("[AWS] ๐Ÿ“‹ Registering AMI from imported snapshot: %s", snapshotID) + registerOutput, err := a.ec2.RegisterImage( + context.TODO(), + &ec2.RegisterImageInput{ + Architecture: ec2Arch, + BootMode: ec2BootMode, + VirtualizationType: aws.String("hvm"), + Name: aws.String(name), + RootDeviceName: aws.String("/dev/sda1"), + EnaSupport: aws.Bool(true), + BlockDeviceMappings: []ec2types.BlockDeviceMapping{ + { + DeviceName: aws.String("/dev/sda1"), + Ebs: &ec2types.EbsBlockDevice{ + SnapshotId: aws.String(snapshotID), + }, + }, + }, + }, + ) + if err != nil { + return "", "", err + } + + imageID := aws.ToString(registerOutput.ImageId) + olog.Printf("[AWS] ๐ŸŽ‰ AMI registered: %s", imageID) + + // Tag the image with the image name. + _, err = a.ec2.CreateTags( + context.TODO(), + &ec2.CreateTagsInput{ + Resources: []string{imageID}, + Tags: []ec2types.Tag{ + { + Key: aws.String("Name"), + Value: aws.String(name), + }, + }, + }, + ) + if err != nil { + return "", "", err + } + + if len(shareWith) > 0 { + err = a.ShareImage(imageID, []string{snapshotID}, shareWith) + if err != nil { + return "", "", err + } + } + + return imageID, snapshotID, nil +} + +func (a *AWS) DeleteObject(bucket, key string) error { + _, err := a.s3.DeleteObject( + context.TODO(), + &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }, + ) + return err +} + +// ShareImage shares the AMI and its associated snapshots with the specified user IDs. +// If no snapshot IDs are provided, it will find the snapshot IDs associated with the AMI. +func (a *AWS) ShareImage(ami string, snapshotIDs, userIDs []string) error { + // If no snapshot IDs are provided, we will try to find the snapshot IDs + // associated with the AMI. + if len(snapshotIDs) == 0 { + imgs, err := a.ec2.DescribeImages( + context.TODO(), + &ec2.DescribeImagesInput{ + ImageIds: []string{ami}, + }, + ) + if err != nil { + return err + } + if len(imgs.Images) == 0 { + return fmt.Errorf("Unable to find image with id: %v", ami) + } + + for _, bdm := range imgs.Images[0].BlockDeviceMappings { + snapshotIDs = append(snapshotIDs, *bdm.Ebs.SnapshotId) + } + } + + for _, snapshotID := range snapshotIDs { + err := a.shareSnapshot(snapshotID, userIDs) + if err != nil { + return err + } + } + + err := a.shareImage(ami, userIDs) + if err != nil { + return err + } + return nil +} + +func (a *AWS) shareImage(ami string, userIDs []string) error { + olog.Println("[AWS] ๐Ÿ’ฟ Sharing ec2 AMI") + var launchPerms []ec2types.LaunchPermission + + for idx := range userIDs { + launchPerms = append(launchPerms, ec2types.LaunchPermission{ + UserId: aws.String(userIDs[idx]), + }) + } + _, err := a.ec2.ModifyImageAttribute( + context.TODO(), + &ec2.ModifyImageAttributeInput{ + ImageId: &ami, + LaunchPermission: &ec2types.LaunchPermissionModifications{ + Add: launchPerms, + }, + }, + ) + if err != nil { + olog.Printf("[AWS] ๐Ÿ“จ Error sharing AMI: %v", err) + return err + } + olog.Println("[AWS] ๐Ÿ’ฟ Shared AMI") + return nil +} + +func (a *AWS) shareSnapshot(snapshotId string, userIds []string) error { + olog.Println("[AWS] ๐ŸŽฅ Sharing ec2 snapshot") + _, err := a.ec2.ModifySnapshotAttribute( + context.TODO(), + &ec2.ModifySnapshotAttributeInput{ + Attribute: ec2types.SnapshotAttributeNameCreateVolumePermission, + OperationType: ec2types.OperationTypeAdd, + SnapshotId: &snapshotId, + UserIds: userIds, + }, + ) + if err != nil { + olog.Printf("[AWS] ๐Ÿ“จ Error sharing ec2 snapshot: %v", err) + return err + } + olog.Println("[AWS] ๐Ÿ“จ Shared ec2 snapshot") + return nil +} + +func (a *AWS) S3ObjectPresignedURL(bucket, objectKey string) (string, error) { + olog.Printf("[AWS] ๐Ÿ“‹ Generating Presigned URL for S3 object %s/%s", bucket, objectKey) + req, err := a.s3presign.PresignGetObject( + context.TODO(), + &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + }, + func(opts *s3.PresignOptions) { + opts.Expires = time.Duration(7 * 24 * time.Hour) + }, + ) + if err != nil { + return "", err + } + + olog.Println("[AWS] ๐ŸŽ‰ S3 Presigned URL ready") + return req.URL, nil +} + +func (a *AWS) MarkS3ObjectAsPublic(bucket, objectKey string) error { + olog.Printf("[AWS] ๐Ÿ‘ Making S3 object public %s/%s", bucket, objectKey) + _, err := a.s3.PutObjectAcl( + context.TODO(), + &s3.PutObjectAclInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + ACL: s3types.ObjectCannedACL(s3types.ObjectCannedACLPublicRead), + }, + ) + if err != nil { + return err + } + + olog.Println("[AWS] โœ”๏ธ Making S3 object public successful") + return nil +} + +func (a *AWS) Regions() ([]string, error) { + out, err := a.ec2.DescribeRegions( + context.TODO(), + &ec2.DescribeRegionsInput{}, + ) + if err != nil { + return nil, err + } + + result := []string{} + for _, r := range out.Regions { + result = append(result, aws.ToString(r.RegionName)) + } + + return result, nil +} + +func (a *AWS) Buckets() ([]string, error) { + out, err := a.s3.ListBuckets( + context.TODO(), + nil, + ) + if err != nil { + return nil, err + } + + result := []string{} + for _, b := range out.Buckets { + result = append(result, aws.ToString(b.Name)) + } + + return result, nil +} + +// checkAWSPermissionMatrix internal helper function, checks if the requiredPermission is +// covered by the currentPermission (consulting the PermissionsMatrix) +func checkAWSPermissionMatrix(requiredPermission s3types.Permission, currentPermission s3types.Permission) bool { + requiredPermissions, exists := S3PermissionsMatrix[requiredPermission] + if !exists { + return false + } + + for _, permission := range requiredPermissions { + if permission == currentPermission { + return true + } + } + return false +} + +// CheckBucketPermission check if the current account (of a.s3) has the `permission` on the given bucket +func (a *AWS) CheckBucketPermission(bucketName string, permission s3types.Permission) (bool, error) { + resp, err := a.s3.GetBucketAcl( + context.TODO(), + &s3.GetBucketAclInput{ + Bucket: aws.String(bucketName), + }, + ) + if err != nil { + return false, err + } + + for _, grant := range resp.Grants { + if checkAWSPermissionMatrix(permission, grant.Permission) { + return true, nil + } + } + return false, nil +} + +func (a *AWS) CreateSecurityGroupEC2(name, description string) (*ec2.CreateSecurityGroupOutput, error) { + return a.ec2.CreateSecurityGroup( + context.TODO(), + &ec2.CreateSecurityGroupInput{ + GroupName: aws.String(name), + Description: aws.String(description), + }, + ) +} + +func (a *AWS) DeleteSecurityGroupEC2(groupID string) (*ec2.DeleteSecurityGroupOutput, error) { + return a.ec2.DeleteSecurityGroup( + context.TODO(), + &ec2.DeleteSecurityGroupInput{ + GroupId: &groupID, + }) +} + +func (a *AWS) AuthorizeSecurityGroupIngressEC2(groupID, address string, from, to int32, proto string) (*ec2.AuthorizeSecurityGroupIngressOutput, error) { + return a.ec2.AuthorizeSecurityGroupIngress( + context.TODO(), + &ec2.AuthorizeSecurityGroupIngressInput{ + CidrIp: aws.String(address), + GroupId: &groupID, + FromPort: aws.Int32(from), + ToPort: aws.Int32(to), + IpProtocol: aws.String(proto), + }) +} + +func (a *AWS) RunInstanceEC2(imageID, secGroupID, userData, instanceType string) (*ec2types.Reservation, error) { + ec2InstanceType := ec2types.InstanceType(instanceType) + if !slices.Contains(ec2InstanceType.Values(), ec2InstanceType) { + return nil, fmt.Errorf("ec2 doesn't support the following instance type: %s", instanceType) + } + + runInstanceOutput, err := a.ec2.RunInstances( + context.TODO(), + &ec2.RunInstancesInput{ + MaxCount: aws.Int32(1), + MinCount: aws.Int32(1), + ImageId: &imageID, + InstanceType: ec2InstanceType, + SecurityGroupIds: []string{secGroupID}, + UserData: aws.String(encodeBase64(userData)), + }) + if err != nil { + return nil, err + } + + if len(runInstanceOutput.Instances) == 0 { + return nil, fmt.Errorf("no instances were created") + } + instanceWaiter := newInstanceRunningWaiterEC2(a.ec2) + err = instanceWaiter.Wait( + context.TODO(), + &ec2.DescribeInstancesInput{ + InstanceIds: []string{aws.ToString(runInstanceOutput.Instances[0].InstanceId)}, + }, + time.Hour, + ) + if err != nil { + return nil, err + } + + reservation, err := a.instanceReservation(aws.ToString(runInstanceOutput.Instances[0].InstanceId)) + if err != nil { + return nil, fmt.Errorf("failed to get reservation for instance %s: %w", aws.ToString(runInstanceOutput.Instances[0].InstanceId), err) + } + + return reservation, nil +} + +func (a *AWS) TerminateInstanceEC2(instanceID string) (*ec2.TerminateInstancesOutput, error) { + // We need to terminate the instance now and wait until the termination is done. + // Otherwise, it wouldn't be possible to delete the image. + res, err := a.ec2.TerminateInstances( + context.TODO(), + &ec2.TerminateInstancesInput{ + InstanceIds: []string{ + instanceID, + }, + }) + if err != nil { + return nil, err + } + + instanceWaiter := newTerminateInstancesWaiterEC2(a.ec2) + err = instanceWaiter.Wait( + context.TODO(), + &ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceID}, + }, + time.Hour, + ) + if err != nil { + return nil, err + } + + return res, nil +} + +func (a *AWS) GetInstanceAddress(instanceID string) (string, error) { + reservation, err := a.instanceReservation(instanceID) + if err != nil { + return "", err + } + + return *reservation.Instances[0].PublicIpAddress, nil +} + +// DeleteEC2Image deletes the specified image and its associated snapshot +func (a *AWS) DeleteEC2Image(imageID, snapshotID string) error { + var retErr error + + // firstly, deregister the image + _, err := a.ec2.DeregisterImage( + context.TODO(), + &ec2.DeregisterImageInput{ + ImageId: &imageID, + }) + + if err != nil { + return err + } + + // now it's possible to delete the snapshot + _, err = a.ec2.DeleteSnapshot( + context.TODO(), + &ec2.DeleteSnapshotInput{ + SnapshotId: &snapshotID, + }) + + if err != nil { + return err + } + + return retErr +} + +func (a *AWS) instanceReservation(id string) (*ec2types.Reservation, error) { + describeInstancesOutput, err := a.ec2.DescribeInstances( + context.TODO(), + &ec2.DescribeInstancesInput{ + InstanceIds: []string{id}, + }, + ) + if err != nil { + return nil, err + } + + if len(describeInstancesOutput.Reservations) == 0 || len(describeInstancesOutput.Reservations[0].Instances) == 0 { + return nil, fmt.Errorf("no reservation found for instance %s", id) + } + + return &describeInstancesOutput.Reservations[0], nil +} + +// encodeBase64 encodes string to base64-encoded string +func encodeBase64(input string) string { + return base64.StdEncoding.EncodeToString([]byte(input)) +} diff --git a/vendor/github.com/osbuild/images/pkg/cloud/awscloud/client_interfaces.go b/vendor/github.com/osbuild/images/pkg/cloud/awscloud/client_interfaces.go new file mode 100644 index 000000000..850f5b2d7 --- /dev/null +++ b/vendor/github.com/osbuild/images/pkg/cloud/awscloud/client_interfaces.go @@ -0,0 +1,67 @@ +package awscloud + +import ( + "context" + "time" + + awsSigner "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type ec2Client interface { + DescribeRegions(context.Context, *ec2.DescribeRegionsInput, ...func(*ec2.Options)) (*ec2.DescribeRegionsOutput, error) + + // Security Groups + AuthorizeSecurityGroupIngress(context.Context, *ec2.AuthorizeSecurityGroupIngressInput, ...func(*ec2.Options)) (*ec2.AuthorizeSecurityGroupIngressOutput, error) + CreateSecurityGroup(context.Context, *ec2.CreateSecurityGroupInput, ...func(*ec2.Options)) (*ec2.CreateSecurityGroupOutput, error) + DeleteSecurityGroup(context.Context, *ec2.DeleteSecurityGroupInput, ...func(*ec2.Options)) (*ec2.DeleteSecurityGroupOutput, error) + + // Instances + DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) + RunInstances(context.Context, *ec2.RunInstancesInput, ...func(*ec2.Options)) (*ec2.RunInstancesOutput, error) + TerminateInstances(context.Context, *ec2.TerminateInstancesInput, ...func(*ec2.Options)) (*ec2.TerminateInstancesOutput, error) + + // Images + RegisterImage(context.Context, *ec2.RegisterImageInput, ...func(*ec2.Options)) (*ec2.RegisterImageOutput, error) + DeregisterImage(context.Context, *ec2.DeregisterImageInput, ...func(*ec2.Options)) (*ec2.DeregisterImageOutput, error) + DescribeImages(context.Context, *ec2.DescribeImagesInput, ...func(*ec2.Options)) (*ec2.DescribeImagesOutput, error) + ModifyImageAttribute(context.Context, *ec2.ModifyImageAttributeInput, ...func(*ec2.Options)) (*ec2.ModifyImageAttributeOutput, error) + + // Snapshots + DeleteSnapshot(context.Context, *ec2.DeleteSnapshotInput, ...func(*ec2.Options)) (*ec2.DeleteSnapshotOutput, error) + DescribeImportSnapshotTasks(context.Context, *ec2.DescribeImportSnapshotTasksInput, ...func(*ec2.Options)) (*ec2.DescribeImportSnapshotTasksOutput, error) + ImportSnapshot(context.Context, *ec2.ImportSnapshotInput, ...func(*ec2.Options)) (*ec2.ImportSnapshotOutput, error) + ModifySnapshotAttribute(context.Context, *ec2.ModifySnapshotAttributeInput, ...func(*ec2.Options)) (*ec2.ModifySnapshotAttributeOutput, error) + + // Tags + CreateTags(context.Context, *ec2.CreateTagsInput, ...func(*ec2.Options)) (*ec2.CreateTagsOutput, error) +} + +type snapshotImportedWaiterEC2 interface { + WaitForOutput(ctx context.Context, params *ec2.DescribeImportSnapshotTasksInput, maxWaitDur time.Duration, optFns ...func(*ec2.SnapshotImportedWaiterOptions)) (*ec2.DescribeImportSnapshotTasksOutput, error) +} + +type instanceRunningWaiterEC2 interface { + Wait(ctx context.Context, params *ec2.DescribeInstancesInput, maxWaitDur time.Duration, optFns ...func(*ec2.InstanceRunningWaiterOptions)) error +} + +type instanceTerminatedWaiterEC2 interface { + Wait(ctx context.Context, params *ec2.DescribeInstancesInput, maxWaitDur time.Duration, optFns ...func(*ec2.InstanceTerminatedWaiterOptions)) error +} + +type s3Client interface { + GetBucketAcl(ctx context.Context, params *s3.GetBucketAclInput, optFns ...func(*s3.Options)) (*s3.GetBucketAclOutput, error) + DeleteObject(context.Context, *s3.DeleteObjectInput, ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) + ListBuckets(context.Context, *s3.ListBucketsInput, ...func(*s3.Options)) (*s3.ListBucketsOutput, error) + PutObjectAcl(context.Context, *s3.PutObjectAclInput, ...func(*s3.Options)) (*s3.PutObjectAclOutput, error) +} + +type s3Uploader interface { + Upload(context.Context, *s3.PutObjectInput, ...func(*manager.Uploader)) (*manager.UploadOutput, error) +} + +type s3Presign interface { + PresignGetObject(context.Context, *s3.GetObjectInput, ...func(*s3.PresignOptions)) (*awsSigner.PresignedHTTPRequest, error) +} diff --git a/vendor/github.com/osbuild/images/pkg/cloud/awscloud/uploader.go b/vendor/github.com/osbuild/images/pkg/cloud/awscloud/uploader.go new file mode 100644 index 000000000..d12501976 --- /dev/null +++ b/vendor/github.com/osbuild/images/pkg/cloud/awscloud/uploader.go @@ -0,0 +1,137 @@ +package awscloud + +import ( + "errors" + "fmt" + "io" + "slices" + + s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/google/uuid" + + "github.com/osbuild/images/pkg/arch" + "github.com/osbuild/images/pkg/cloud" + "github.com/osbuild/images/pkg/platform" +) + +type awsUploader struct { + client awsClient + + region string + bucketName string + imageName string + targetArch arch.Arch + bootMode *platform.BootMode +} + +type UploaderOptions struct { + TargetArch arch.Arch + // BootMode to set for the AMI. If nil, no explicit boot mode will be set. + BootMode *platform.BootMode +} + +// testing support +type awsClient interface { + Regions() ([]string, error) + Buckets() ([]string, error) + CheckBucketPermission(string, s3types.Permission) (bool, error) + UploadFromReader(io.Reader, string, string) (*s3manager.UploadOutput, error) + Register(name, bucket, key string, shareWith []string, architecture arch.Arch, bootMode *platform.BootMode, importRole *string) (string, string, error) + DeleteObject(string, string) error +} + +var newAwsClient = func(region string) (awsClient, error) { + return NewDefault(region) +} + +func NewUploader(region, bucketName, imageName string, opts *UploaderOptions) (cloud.Uploader, error) { + if opts == nil { + opts = &UploaderOptions{} + } + client, err := newAwsClient(region) + if err != nil { + return nil, err + } + + return &awsUploader{ + client: client, + region: region, + bucketName: bucketName, + imageName: imageName, + targetArch: opts.TargetArch, + bootMode: opts.BootMode, + }, nil +} + +var _ cloud.Uploader = &awsUploader{} + +func (au *awsUploader) Check(status io.Writer) error { + fmt.Fprintf(status, "Checking AWS region access...\n") + regions, err := au.client.Regions() + if err != nil { + return fmt.Errorf("retrieving AWS regions for '%s' failed: %w", au.region, err) + } + + if !slices.Contains(regions, au.region) { + return fmt.Errorf("given AWS region '%s' not found", au.region) + } + + fmt.Fprintf(status, "Checking AWS bucket...\n") + buckets, err := au.client.Buckets() + if err != nil { + return fmt.Errorf("retrieving AWS list of buckets failed: %w", err) + } + if !slices.Contains(buckets, au.bucketName) { + return fmt.Errorf("bucket '%s' not found in the given AWS account", au.bucketName) + } + + fmt.Fprintf(status, "Checking AWS bucket permissions...\n") + writePermission, err := au.client.CheckBucketPermission(au.bucketName, s3types.PermissionWrite) + if err != nil { + return err + } + if !writePermission { + return fmt.Errorf("you don't have write permissions to bucket '%s' with the given AWS account", au.bucketName) + } + fmt.Fprintf(status, "Upload conditions met.\n") + return nil +} + +func (au *awsUploader) UploadAndRegister(r io.Reader, status io.Writer) (err error) { + keyName := fmt.Sprintf("%s-%s", uuid.New().String(), au.imageName) + fmt.Fprintf(status, "Uploading %s to %s:%s\n", au.imageName, au.bucketName, keyName) + + res, err := au.client.UploadFromReader(r, au.bucketName, keyName) + if err != nil { + return err + } + defer func() { + if err != nil { + aErr := au.client.DeleteObject(au.bucketName, keyName) + fmt.Fprintf(status, "Deleted S3 object %s:%s\n", au.bucketName, keyName) + err = errors.Join(err, aErr) + } + }() + fmt.Fprintf(status, "File uploaded to %s\n", res.Location) + if au.targetArch == arch.ARCH_UNSET { + au.targetArch = arch.Current() + } + + fmt.Fprintf(status, "Registering AMI %s\n", au.imageName) + ami, snapshot, err := au.client.Register(au.imageName, au.bucketName, keyName, nil, au.targetArch, au.bootMode, nil) + if err != nil { + return err + } + + fmt.Fprintf(status, "Deleted S3 object %s:%s\n", au.bucketName, keyName) + if err := au.client.DeleteObject(au.bucketName, keyName); err != nil { + return err + } + fmt.Fprintf(status, "AMI registered: %s\nSnapshot ID: %s\n", ami, snapshot) + if err != nil { + return err + } + + return nil +} diff --git a/vendor/github.com/osbuild/images/pkg/cloud/uploader.go b/vendor/github.com/osbuild/images/pkg/cloud/uploader.go new file mode 100644 index 000000000..2378028b4 --- /dev/null +++ b/vendor/github.com/osbuild/images/pkg/cloud/uploader.go @@ -0,0 +1,24 @@ +package cloud + +import ( + "io" +) + +// Uploader is an interface that is returned from the actual +// cloud implementation. The uploader will be parameterized +// by the actual cloud implemntation, e.g. +// +// awscloud.NewUploader(region, bucket, image) Uploader +// +// which is outside the scope of this interface. +type Uploader interface { + // Check can be called before the actual upload to ensure + // all permissions are correct + Check(status io.Writer) error + + // UploadAndRegister will upload the given image from + // the reader and write status message to the given + // status writer. + // To implement progress a proxy reader can be used. + UploadAndRegister(f io.Reader, status io.Writer) error +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 03d9cee29..ebb51a1c3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -972,6 +972,8 @@ github.com/osbuild/images/internal/workload github.com/osbuild/images/pkg/arch github.com/osbuild/images/pkg/artifact github.com/osbuild/images/pkg/cert +github.com/osbuild/images/pkg/cloud +github.com/osbuild/images/pkg/cloud/awscloud github.com/osbuild/images/pkg/cloud/azure github.com/osbuild/images/pkg/container github.com/osbuild/images/pkg/crypt