diff --git a/cmd/osbuild-image-tests/main_test.go b/cmd/osbuild-image-tests/main_test.go index 90470b637..f1403da9f 100644 --- a/cmd/osbuild-image-tests/main_test.go +++ b/cmd/osbuild-image-tests/main_test.go @@ -328,19 +328,16 @@ func testBootUsingAWS(t *testing.T, imagePath string) { imageName, err := test.GenerateCIArtifactName("osbuild-image-tests-image-") require.NoError(t, err) - e, err := boot.NewEC2(creds) - require.NoError(t, err) - // the following line should be done by osbuild-composer at some point err = boot.UploadImageToAWS(creds, imagePath, imageName) require.NoErrorf(t, err, "upload to amazon failed, resources could have been leaked") - imageDesc, err := boot.DescribeEC2Image(e, imageName) + imageDesc, err := boot.DescribeEC2Image(creds, imageName) require.NoErrorf(t, err, "cannot describe the ec2 image") // delete the image after the test is over defer func() { - err = boot.DeleteEC2Image(e, imageDesc) + err = boot.DeleteEC2Image(creds, imageDesc) require.NoErrorf(t, err, "cannot delete the ec2 image, resources could have been leaked") }() @@ -359,7 +356,7 @@ func testBootUsingAWS(t *testing.T, imagePath string) { // boot the uploaded image and try to connect to it err = boot.WithSSHKeyPair(func(privateKey, publicKey string) error { - return boot.WithBootedImageInEC2(e, securityGroupName, imageDesc, publicKey, instanceType, func(address string) error { + return boot.WithBootedImageInEC2(creds, securityGroupName, imageDesc, publicKey, instanceType, func(address string) error { testSSH(t, address, privateKey, nil) return nil }) diff --git a/internal/boot/aws.go b/internal/boot/aws.go index 4b155510e..43d947dee 100644 --- a/internal/boot/aws.go +++ b/internal/boot/aws.go @@ -3,15 +3,17 @@ package boot import ( + "context" "encoding/base64" "errors" "fmt" "os" + "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/osbuild/images/pkg/arch" "github.com/osbuild/osbuild-composer/internal/cloud/awscloud" @@ -92,57 +94,40 @@ func wrapErrorf(innerError error, format string, a ...interface{}) error { func UploadImageToAWS(c *awsCredentials, imagePath string, imageName string) error { uploader, err := awscloud.New(c.Region, c.AccessKeyId, c.SecretAccessKey, c.sessionToken) if err != nil { - return fmt.Errorf("cannot create aws uploader: %v", err) + return fmt.Errorf("cannot create aws uploader: %w", err) } _, err = uploader.Upload(imagePath, c.Bucket, imageName) if err != nil { - return fmt.Errorf("cannot upload the image: %v", err) + return fmt.Errorf("cannot upload the image: %w", err) } _, err = uploader.Register(imageName, c.Bucket, imageName, nil, arch.Current().String(), nil) if err != nil { - return fmt.Errorf("cannot register the image: %v", err) + return fmt.Errorf("cannot register the image: %w", err) } return nil } -// NewEC2 creates EC2 struct from given credentials -func NewEC2(c *awsCredentials) (*ec2.EC2, error) { - creds := credentials.NewStaticCredentials(c.AccessKeyId, c.SecretAccessKey, "") - sess, err := session.NewSession(&aws.Config{ - Credentials: creds, - Region: aws.String(c.Region), - }) - if err != nil { - return nil, fmt.Errorf("cannot create aws session: %v", err) - } - - return ec2.New(sess), nil -} - type imageDescription struct { Id *string SnapshotId *string // this doesn't support multiple snapshots per one image, // because this feature is not supported in composer + Img *ec2types.Image } // DescribeEC2Image searches for EC2 image by its name and returns // its id and snapshot id -func DescribeEC2Image(e *ec2.EC2, imageName string) (*imageDescription, error) { - imageDescriptions, err := e.DescribeImages(&ec2.DescribeImagesInput{ - Filters: []*ec2.Filter{ - { - Name: aws.String("name"), - Values: []*string{ - aws.String(imageName), - }, - }, - }, - }) +func DescribeEC2Image(c *awsCredentials, imageName string) (*imageDescription, error) { + awscl, err := awscloud.New(c.Region, c.AccessKeyId, c.SecretAccessKey, c.sessionToken) if err != nil { - return nil, fmt.Errorf("cannot describe the image: %v", err) + return nil, fmt.Errorf("cannot create aws client: %w", err) + } + + imageDescriptions, err := awscl.DescribeImagesByName(imageName) + if err != nil { + return nil, fmt.Errorf("cannot describe images: %w", err) } imageId := imageDescriptions.Images[0].ImageId snapshotId := imageDescriptions.Images[0].BlockDeviceMappings[0].Ebs.SnapshotId @@ -150,37 +135,27 @@ func DescribeEC2Image(e *ec2.EC2, imageName string) (*imageDescription, error) { return &imageDescription{ Id: imageId, SnapshotId: snapshotId, + Img: &imageDescriptions.Images[0], }, nil } // DeleteEC2Image deletes the specified image and its associated snapshot -func DeleteEC2Image(e *ec2.EC2, imageDesc *imageDescription) error { - var retErr error - - // firstly, deregister the image - _, err := e.DeregisterImage(&ec2.DeregisterImageInput{ - ImageId: imageDesc.Id, - }) - +func DeleteEC2Image(c *awsCredentials, imageDesc *imageDescription) error { + awscl, err := awscloud.New(c.Region, c.AccessKeyId, c.SecretAccessKey, c.sessionToken) if err != nil { - retErr = wrapErrorf(retErr, "cannot deregister the image: %v", err) + return fmt.Errorf("cannot create aws client: %w", err) } - - // now it's possible to delete the snapshot - _, err = e.DeleteSnapshot(&ec2.DeleteSnapshotInput{ - SnapshotId: imageDesc.SnapshotId, - }) - - if err != nil { - retErr = wrapErrorf(retErr, "cannot delete the snapshot: %v", err) - } - - return retErr + return awscl.RemoveSnapshotAndDeregisterImage(imageDesc.Img) } // WithBootedImageInEC2 runs the function f in the context of booted // image in AWS EC2 -func WithBootedImageInEC2(e *ec2.EC2, securityGroupName string, imageDesc *imageDescription, publicKey string, instanceType string, f func(address string) error) (retErr error) { +func WithBootedImageInEC2(c *awsCredentials, securityGroupName string, imageDesc *imageDescription, publicKey string, instanceType string, f func(address string) error) (retErr error) { + awscl, err := awscloud.New(c.Region, c.AccessKeyId, c.SecretAccessKey, c.sessionToken) + if err != nil { + return fmt.Errorf("cannot create aws client: %w", err) + } + // generate user data with given public key userData, err := CreateUserData(publicKey) if err != nil { @@ -190,71 +165,92 @@ func WithBootedImageInEC2(e *ec2.EC2, securityGroupName string, imageDesc *image // Security group must be now generated, because by default // all traffic to EC2 instance is filtered. // Firstly create a security group - securityGroup, err := e.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{ - GroupName: aws.String(securityGroupName), - Description: aws.String("image-tests-security-group"), - }) + securityGroup, err := awscl.EC2ForTestsOnly().CreateSecurityGroup( + context.Background(), + &ec2.CreateSecurityGroupInput{ + GroupName: aws.String(securityGroupName), + Description: aws.String("image-tests-security-group"), + }, + ) if err != nil { - return fmt.Errorf("cannot create a new security group: %v", err) + return fmt.Errorf("cannot create a new security group: %w", err) } defer func() { - _, err = e.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{ - GroupId: securityGroup.GroupId, - }) + _, err = awscl.EC2ForTestsOnly().DeleteSecurityGroup( + context.Background(), + &ec2.DeleteSecurityGroupInput{ + GroupId: securityGroup.GroupId, + }, + ) if err != nil { - retErr = wrapErrorf(retErr, "cannot delete the security group: %v", err) + retErr = wrapErrorf(retErr, "cannot delete the security group: %w", err) } }() // Authorize incoming SSH connections. - _, err = e.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ - CidrIp: aws.String("0.0.0.0/0"), - GroupId: securityGroup.GroupId, - FromPort: aws.Int64(22), - ToPort: aws.Int64(22), - IpProtocol: aws.String("tcp"), - }) + _, err = awscl.EC2ForTestsOnly().AuthorizeSecurityGroupIngress( + context.Background(), + &ec2.AuthorizeSecurityGroupIngressInput{ + CidrIp: aws.String("0.0.0.0/0"), + GroupId: securityGroup.GroupId, + FromPort: aws.Int32(22), + ToPort: aws.Int32(22), + IpProtocol: aws.String("tcp"), + }, + ) if err != nil { - return fmt.Errorf("canot add a rule to the security group: %v", err) + return fmt.Errorf("canot add a rule to the security group: %w", err) } // Finally, run the instance from the given image and with the created security group - res, err := e.RunInstances(&ec2.RunInstancesInput{ - MaxCount: aws.Int64(1), - MinCount: aws.Int64(1), - ImageId: imageDesc.Id, - InstanceType: aws.String(instanceType), - SecurityGroupIds: []*string{securityGroup.GroupId}, - UserData: aws.String(encodeBase64(userData)), - }) + res, err := awscl.EC2ForTestsOnly().RunInstances( + context.Background(), + &ec2.RunInstancesInput{ + MaxCount: aws.Int32(1), + MinCount: aws.Int32(1), + ImageId: imageDesc.Id, + InstanceType: ec2types.InstanceType(instanceType), + SecurityGroupIds: []string{*securityGroup.GroupId}, + UserData: aws.String(encodeBase64(userData)), + }, + ) if err != nil { - return fmt.Errorf("cannot create a new instance: %v", err) + return fmt.Errorf("cannot create a new instance: %w", err) } describeInstanceInput := &ec2.DescribeInstancesInput{ - InstanceIds: []*string{ - res.Instances[0].InstanceId, + InstanceIds: []string{ + *res.Instances[0].InstanceId, }, } defer func() { // We need to terminate the instance now and wait until the termination is done. // Otherwise, it wouldn't be possible to delete the image. - _, err = e.TerminateInstances(&ec2.TerminateInstancesInput{ - InstanceIds: []*string{ - res.Instances[0].InstanceId, + _, err = awscl.EC2ForTestsOnly().TerminateInstances( + context.Background(), + &ec2.TerminateInstancesInput{ + InstanceIds: []string{ + *res.Instances[0].InstanceId, + }, }, - }) + ) if err != nil { - retErr = wrapErrorf(retErr, "cannot terminate the instance: %v", err) + retErr = wrapErrorf(retErr, "cannot terminate the instance: %w", err) return } - err = e.WaitUntilInstanceTerminated(describeInstanceInput) + instTermWaiter := ec2.NewInstanceTerminatedWaiter(awscl.EC2ForTestsOnly()) + err = instTermWaiter.Wait( + context.Background(), + describeInstanceInput, + time.Hour, + ) if err != nil { - retErr = wrapErrorf(retErr, "waiting for the instance termination failed: %v", err) + retErr = wrapErrorf(retErr, "cannot terminate the instance: %w", err) + return } }() @@ -262,15 +258,20 @@ func WithBootedImageInEC2(e *ec2.EC2, securityGroupName string, imageDesc *image // is in the state "EXISTS". However, in this state the instance is not // much usable, therefore wait until "RUNNING" state, in which the instance // actually can do something useful for us. - err = e.WaitUntilInstanceRunning(describeInstanceInput) + instWaiter := ec2.NewInstanceRunningWaiter(awscl.EC2ForTestsOnly()) + err = instWaiter.Wait( + context.Background(), + describeInstanceInput, + time.Hour, + ) if err != nil { - return fmt.Errorf("waiting for the instance to be running failed: %v", err) + return fmt.Errorf("waiting for the instance to be running failed: %w", err) } // By describing the instance, we can get the ip address. - out, err := e.DescribeInstances(describeInstanceInput) + out, err := awscl.EC2ForTestsOnly().DescribeInstances(context.Background(), describeInstanceInput) if err != nil { - return fmt.Errorf("cannot describe the instance: %v", err) + return fmt.Errorf("cannot describe the instance: %w", err) } return f(*out.Reservations[0].Instances[0].PublicIpAddress) diff --git a/internal/cloud/awscloud/awscloud.go b/internal/cloud/awscloud/awscloud.go index 729820282..83575159f 100644 --- a/internal/cloud/awscloud/awscloud.go +++ b/internal/cloud/awscloud/awscloud.go @@ -180,6 +180,11 @@ func NewForEndpointFromFile(filename, endpoint, region, caBundle string, skipSSL return newAwsFromCredsWithEndpoint(config.WithSharedCredentialsFiles([]string{filename, "default"}), region, endpoint, caBundle, skipSSLVerification) } +// This is used by the internal/boot test, which access the ec2 apis directly +func (a *AWS) EC2ForTestsOnly() EC2 { + return a.ec2 +} + func (a *AWS) Upload(filename, bucket, key string) (*manager.UploadOutput, error) { file, err := os.Open(filename) if err != nil { @@ -627,3 +632,19 @@ func (a *AWS) Regions() ([]string, error) { } return result, nil } + +func (a *AWS) DescribeImagesByName(name string) (*ec2.DescribeImagesOutput, error) { + return a.ec2.DescribeImages( + context.Background(), + &ec2.DescribeImagesInput{ + Filters: []ec2types.Filter{ + { + Name: aws.String("name"), + Values: []string{ + name, + }, + }, + }, + }, + ) +} diff --git a/internal/cloud/awscloud/client-interfaces.go b/internal/cloud/awscloud/client-interfaces.go index 067bad41e..698838c97 100644 --- a/internal/cloud/awscloud/client-interfaces.go +++ b/internal/cloud/awscloud/client-interfaces.go @@ -32,6 +32,7 @@ type EC2 interface { // Instances DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) DescribeInstanceStatus(context.Context, *ec2.DescribeInstanceStatusInput, ...func(*ec2.Options)) (*ec2.DescribeInstanceStatusOutput, error) + RunInstances(context.Context, *ec2.RunInstancesInput, ...func(*ec2.Options)) (*ec2.RunInstancesOutput, error) TerminateInstances(context.Context, *ec2.TerminateInstancesInput, ...func(*ec2.Options)) (*ec2.TerminateInstancesOutput, error) // Fleets diff --git a/internal/cloud/awscloud/mocks_test.go b/internal/cloud/awscloud/mocks_test.go index 67a759343..00757d43a 100644 --- a/internal/cloud/awscloud/mocks_test.go +++ b/internal/cloud/awscloud/mocks_test.go @@ -261,6 +261,10 @@ func (m *ec2mock) DescribeInstanceStatus(ctx context.Context, input *ec2.Describ }, nil } +func (m *ec2mock) RunInstances(ctx context.Context, input *ec2.RunInstancesInput, optfns ...func(*ec2.Options)) (*ec2.RunInstancesOutput, error) { + return nil, nil +} + func (m *ec2mock) TerminateInstances(ctx context.Context, input *ec2.TerminateInstancesInput, optfns ...func(*ec2.Options)) (*ec2.TerminateInstancesOutput, error) { m.calledFn["TerminateInstances"] += 1 return nil, nil