cloud/awscloud: switch ec2 to v2 sdk
This commit is contained in:
Sanne Raymaekers
parent
8d158f6031
commit
810e9133e8
3 changed files with 434 additions and 406 deletions
|
|
@ -8,27 +8,24 @@ import (
|
|||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
credentialsv2 "github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/ec2"
|
||||
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/credentials"
|
||||
"github.com/aws/aws-sdk-go/aws/ec2metadata"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/sirupsen/logrus"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
type AWS struct {
|
||||
ec2 *ec2.EC2
|
||||
ec2metadata *ec2metadata.EC2Metadata
|
||||
s3 S3
|
||||
s3uploader S3Manager
|
||||
s3presign S3Presign
|
||||
ec2 EC2
|
||||
ec2imds EC2Imds
|
||||
s3 S3
|
||||
s3uploader S3Manager
|
||||
s3presign S3Presign
|
||||
}
|
||||
|
||||
func newForTest(s3cli S3, upldr S3Manager, sign S3Presign) *AWS {
|
||||
|
|
@ -40,43 +37,30 @@ func newForTest(s3cli S3, upldr S3Manager, sign S3Presign) *AWS {
|
|||
}
|
||||
|
||||
// Create a new session from the credentials and the region and returns an *AWS object initialized with it.
|
||||
func newAwsFromCreds(creds *credentials.Credentials, region string) (*AWS, error) {
|
||||
// Create a Session with a custom region
|
||||
sess, err := session.NewSession(&aws.Config{
|
||||
Credentials: creds,
|
||||
Region: aws.String(region),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
credsValue, err := creds.Get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg, err := config.LoadDefaultConfig(
|
||||
context.Background(),
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentialsv2.NewStaticCredentialsProvider(
|
||||
credsValue.AccessKeyID,
|
||||
credsValue.SecretAccessKey,
|
||||
credsValue.SessionToken,
|
||||
)),
|
||||
)
|
||||
|
||||
// /creds credentials.StaticCredentialsProvider, region string
|
||||
func newAwsFromConfig(cfg aws.Config) *AWS {
|
||||
s3cli := s3.NewFromConfig(cfg)
|
||||
return &AWS{
|
||||
ec2: ec2.New(sess),
|
||||
ec2metadata: ec2metadata.New(sess),
|
||||
s3: s3cli,
|
||||
s3uploader: manager.NewUploader(s3cli),
|
||||
s3presign: s3.NewPresignClient(s3cli),
|
||||
}, nil
|
||||
ec2: ec2.NewFromConfig(cfg),
|
||||
ec2imds: imds.NewFromConfig(cfg),
|
||||
s3: s3cli,
|
||||
s3uploader: manager.NewUploader(s3cli),
|
||||
s3presign: s3.NewPresignClient(s3cli),
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize a new AWS object from individual bits. SessionToken is optional
|
||||
func New(region string, accessKeyID string, accessKey string, sessionToken string) (*AWS, error) {
|
||||
return newAwsFromCreds(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region)
|
||||
cfg, err := config.LoadDefaultConfig(
|
||||
context.Background(),
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKey, sessionToken)),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aws := newAwsFromConfig(cfg)
|
||||
return aws, nil
|
||||
}
|
||||
|
||||
// Initializes a new AWS object with the credentials info found at filename's location.
|
||||
|
|
@ -89,21 +73,40 @@ func New(region string, accessKeyID string, accessKey string, sessionToken strin
|
|||
// "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to
|
||||
// $HOME/.aws/credentials.
|
||||
func NewFromFile(filename string, region string) (*AWS, error) {
|
||||
return newAwsFromCreds(credentials.NewSharedCredentials(filename, "default"), region)
|
||||
cfg, err := config.LoadDefaultConfig(
|
||||
context.Background(),
|
||||
config.WithRegion(region),
|
||||
config.WithSharedCredentialsFiles([]string{
|
||||
filename,
|
||||
"default",
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aws := newAwsFromConfig(cfg)
|
||||
return aws, nil
|
||||
}
|
||||
|
||||
// Initialize a new AWS object from defaults.
|
||||
// Looks for env variables, shared credential file, and EC2 Instance Roles.
|
||||
func NewDefault(region string) (*AWS, error) {
|
||||
return newAwsFromCreds(nil, region)
|
||||
cfg, err := config.LoadDefaultConfig(
|
||||
context.Background(),
|
||||
config.WithRegion(region),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aws := newAwsFromConfig(cfg)
|
||||
return aws, nil
|
||||
}
|
||||
|
||||
func RegionFromInstanceMetadata() (string, error) {
|
||||
sess, err := session.NewSession()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
identity, err := ec2metadata.New(sess).GetInstanceIdentityDocument()
|
||||
identity, err := imds.New(imds.Options{}).GetInstanceIdentityDocument(
|
||||
context.Background(),
|
||||
&imds.GetInstanceIdentityDocumentInput{},
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
@ -111,29 +114,11 @@ func RegionFromInstanceMetadata() (string, error) {
|
|||
}
|
||||
|
||||
// Create a new session from the credentials and the region and returns an *AWS object initialized with it.
|
||||
func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoint, caBundle string, skipSSLVerification bool) (*AWS, error) {
|
||||
func newAwsFromCredsWithEndpoint(creds config.LoadOptionsFunc, region, endpoint, caBundle string, skipSSLVerification bool) (*AWS, error) {
|
||||
// Create a Session with a custom region
|
||||
s3ForcePathStyle := true
|
||||
sessionOptions := session.Options{
|
||||
Config: aws.Config{
|
||||
Credentials: creds,
|
||||
Region: aws.String(region),
|
||||
Endpoint: &endpoint,
|
||||
S3ForcePathStyle: &s3ForcePathStyle,
|
||||
},
|
||||
}
|
||||
|
||||
credsValue, err := creds.Get()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v2OptionFuncs := []func(*config.LoadOptions) error{
|
||||
config.WithRegion(region),
|
||||
config.WithCredentialsProvider(credentialsv2.NewStaticCredentialsProvider(
|
||||
credsValue.AccessKeyID,
|
||||
credsValue.SecretAccessKey,
|
||||
credsValue.SessionToken,
|
||||
)),
|
||||
creds,
|
||||
}
|
||||
|
||||
if caBundle != "" {
|
||||
|
|
@ -142,30 +127,24 @@ func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoin
|
|||
return nil, err
|
||||
}
|
||||
defer caBundleReader.Close()
|
||||
sessionOptions.CustomCABundle = caBundleReader
|
||||
v2OptionFuncs = append(v2OptionFuncs, config.WithCustomCABundle(caBundleReader))
|
||||
}
|
||||
|
||||
if skipSSLVerification {
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // #nosec G402
|
||||
sessionOptions.Config.HTTPClient = &http.Client{
|
||||
Transport: transport,
|
||||
}
|
||||
v2OptionFuncs = append(v2OptionFuncs, config.WithHTTPClient(&http.Client{
|
||||
Transport: transport,
|
||||
}))
|
||||
}
|
||||
|
||||
sess, err := session.NewSessionWithOptions(sessionOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg, err := config.LoadDefaultConfig(
|
||||
context.Background(),
|
||||
v2OptionFuncs...,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s3cli := s3.NewFromConfig(cfg, func(options *s3.Options) {
|
||||
options.BaseEndpoint = aws.String(endpoint)
|
||||
|
|
@ -173,17 +152,17 @@ func newAwsFromCredsWithEndpoint(creds *credentials.Credentials, region, endpoin
|
|||
})
|
||||
|
||||
return &AWS{
|
||||
ec2: ec2.New(sess),
|
||||
ec2metadata: ec2metadata.New(sess),
|
||||
s3: s3cli,
|
||||
s3uploader: manager.NewUploader(s3cli),
|
||||
s3presign: s3.NewPresignClient(s3cli),
|
||||
ec2: ec2.NewFromConfig(cfg),
|
||||
ec2imds: imds.NewFromConfig(cfg),
|
||||
s3: s3cli,
|
||||
s3uploader: manager.NewUploader(s3cli),
|
||||
s3presign: s3.NewPresignClient(s3cli),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Initialize a new AWS object targeting a specific endpoint from individual bits. SessionToken is optional
|
||||
func NewForEndpoint(endpoint, region, accessKeyID, accessKey, sessionToken, caBundle string, skipSSLVerification bool) (*AWS, error) {
|
||||
return newAwsFromCredsWithEndpoint(credentials.NewStaticCredentials(accessKeyID, accessKey, sessionToken), region, endpoint, caBundle, skipSSLVerification)
|
||||
return newAwsFromCredsWithEndpoint(config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, accessKey, sessionToken)), region, endpoint, caBundle, skipSSLVerification)
|
||||
}
|
||||
|
||||
// Initializes a new AWS object targeting a specific endpoint with the credentials info found at filename's location.
|
||||
|
|
@ -196,7 +175,7 @@ func NewForEndpoint(endpoint, region, accessKeyID, accessKey, sessionToken, caBu
|
|||
// "AWS_SHARED_CREDENTIALS_FILE" env variable or will default to
|
||||
// $HOME/.aws/credentials.
|
||||
func NewForEndpointFromFile(filename, endpoint, region, caBundle string, skipSSLVerification bool) (*AWS, error) {
|
||||
return newAwsFromCredsWithEndpoint(credentials.NewSharedCredentials(filename, "default"), region, endpoint, caBundle, skipSSLVerification)
|
||||
return newAwsFromCredsWithEndpoint(config.WithSharedCredentialsFiles([]string{filename, "default"}), region, endpoint, caBundle, skipSSLVerification)
|
||||
}
|
||||
|
||||
func (a *AWS) Upload(filename, bucket, key string) (*manager.UploadOutput, error) {
|
||||
|
|
@ -223,63 +202,6 @@ func (a *AWS) Upload(filename, bucket, key string) (*manager.UploadOutput, error
|
|||
)
|
||||
}
|
||||
|
||||
// WaitUntilImportSnapshotCompleted uses the Amazon EC2 API operation
|
||||
// DescribeImportSnapshots to wait for a condition to be met before returning.
|
||||
// If the condition is not met within the max attempt window, an error will
|
||||
// be returned.
|
||||
func WaitUntilImportSnapshotTaskCompleted(c *ec2.EC2, input *ec2.DescribeImportSnapshotTasksInput) error {
|
||||
return WaitUntilImportSnapshotTaskCompletedWithContext(c, aws.BackgroundContext(), input)
|
||||
}
|
||||
|
||||
// WaitUntilImportSnapshotCompletedWithContext is an extended version of
|
||||
// WaitUntilImportSnapshotCompleted. With the support for passing in a
|
||||
// context and options to configure the Waiter and the underlying request
|
||||
// options.
|
||||
//
|
||||
// The context must be non-nil and will be used for request cancellation. If
|
||||
// the context is nil a panic will occur. In the future the SDK may create
|
||||
// sub-contexts for http.Requests. See https://golang.org/pkg/context/
|
||||
// for more information on using Contexts.
|
||||
//
|
||||
// NOTE(mhayden): The MaxAttempts is set to zero here so that we will keep
|
||||
// checking the status of the image import until it succeeds or fails. This
|
||||
// process can take anywhere from 5 to 60+ minutes depending on how quickly
|
||||
// AWS can import the snapshot.
|
||||
func WaitUntilImportSnapshotTaskCompletedWithContext(c *ec2.EC2, ctx aws.Context, input *ec2.DescribeImportSnapshotTasksInput, opts ...request.WaiterOption) error {
|
||||
w := request.Waiter{
|
||||
Name: "WaitUntilImportSnapshotTaskCompleted",
|
||||
MaxAttempts: 0,
|
||||
Delay: request.ConstantWaiterDelay(15 * time.Second),
|
||||
Acceptors: []request.WaiterAcceptor{
|
||||
{
|
||||
State: request.SuccessWaiterState,
|
||||
Matcher: request.PathAllWaiterMatch, Argument: "ImportSnapshotTasks[].SnapshotTaskDetail.Status",
|
||||
Expected: "completed",
|
||||
},
|
||||
{
|
||||
State: request.FailureWaiterState,
|
||||
Matcher: request.PathAllWaiterMatch, Argument: "ImportSnapshotTasks[].SnapshotTaskDetail.Status",
|
||||
Expected: "deleted",
|
||||
},
|
||||
},
|
||||
Logger: c.Config.Logger,
|
||||
NewRequest: func(opts []request.Option) (*request.Request, error) {
|
||||
var inCpy *ec2.DescribeImportSnapshotTasksInput
|
||||
if input != nil {
|
||||
tmp := *input
|
||||
inCpy = &tmp
|
||||
}
|
||||
req, _ := c.DescribeImportSnapshotTasksRequest(inCpy)
|
||||
req.SetContext(ctx)
|
||||
req.ApplyOptions(opts...)
|
||||
return req, nil
|
||||
},
|
||||
}
|
||||
w.ApplyOptions(opts...)
|
||||
|
||||
return w.WaitWithContext(ctx)
|
||||
}
|
||||
|
||||
// Register is a function that imports a snapshot, waits for the snapshot to
|
||||
// fully import, tags the snapshot, cleans up the image in S3, and registers
|
||||
// an AMI in AWS.
|
||||
|
|
@ -287,9 +209,9 @@ func WaitUntilImportSnapshotTaskCompletedWithContext(c *ec2.EC2, ctx aws.Context
|
|||
// mode is not specified, then the instances launched from this AMI use the
|
||||
// default boot mode value of the instance type.
|
||||
func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch string, bootMode *string) (*string, error) {
|
||||
rpmArchToEC2Arch := map[string]string{
|
||||
"x86_64": "x86_64",
|
||||
"aarch64": "arm64",
|
||||
rpmArchToEC2Arch := map[string]ec2types.ArchitectureValues{
|
||||
"x86_64": ec2types.ArchitectureValuesX8664,
|
||||
"aarch64": ec2types.ArchitectureValuesArm64,
|
||||
}
|
||||
|
||||
ec2Arch, validArch := rpmArchToEC2Arch[rpmArch]
|
||||
|
|
@ -297,19 +219,28 @@ func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch str
|
|||
return nil, fmt.Errorf("ec2 doesn't support the following arch: %s", rpmArch)
|
||||
}
|
||||
|
||||
bootModeToEC2BootMode := map[string]ec2types.BootModeValues{
|
||||
string(ec2types.BootModeValuesLegacyBios): ec2types.BootModeValuesLegacyBios,
|
||||
string(ec2types.BootModeValuesUefi): ec2types.BootModeValuesUefi,
|
||||
string(ec2types.BootModeValuesUefiPreferred): ec2types.BootModeValuesUefiPreferred,
|
||||
}
|
||||
ec2BootMode := ec2types.BootModeValuesUefiPreferred
|
||||
if bootMode != nil {
|
||||
if !slices.Contains(ec2.BootModeValues_Values(), *bootMode) {
|
||||
bm, validBootMode := bootModeToEC2BootMode[*bootMode]
|
||||
if !validBootMode {
|
||||
return nil, fmt.Errorf("ec2 doesn't support the following boot mode: %s", *bootMode)
|
||||
}
|
||||
ec2BootMode = bm
|
||||
}
|
||||
|
||||
logrus.Infof("[AWS] 📥 Importing snapshot from image: %s/%s", bucket, key)
|
||||
snapshotDescription := fmt.Sprintf("Image Builder AWS Import of %s", name)
|
||||
importTaskOutput, err := a.ec2.ImportSnapshot(
|
||||
context.Background(),
|
||||
&ec2.ImportSnapshotInput{
|
||||
Description: aws.String(snapshotDescription),
|
||||
DiskContainer: &ec2.SnapshotDiskContainer{
|
||||
UserBucket: &ec2.UserBucket{
|
||||
DiskContainer: &ec2types.SnapshotDiskContainer{
|
||||
UserBucket: &ec2types.UserBucket{
|
||||
S3Bucket: aws.String(bucket),
|
||||
S3Key: aws.String(key),
|
||||
},
|
||||
|
|
@ -322,18 +253,27 @@ func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch str
|
|||
}
|
||||
|
||||
logrus.Infof("[AWS] 🚚 Waiting for snapshot to finish importing: %s", *importTaskOutput.ImportTaskId)
|
||||
err = WaitUntilImportSnapshotTaskCompleted(
|
||||
a.ec2,
|
||||
|
||||
// importTaskOutput.
|
||||
snapWaiter := ec2.NewSnapshotImportedWaiter(a.ec2)
|
||||
snapWaitOutput, err := snapWaiter.WaitForOutput(
|
||||
context.Background(),
|
||||
&ec2.DescribeImportSnapshotTasksInput{
|
||||
ImportTaskIds: []*string{
|
||||
importTaskOutput.ImportTaskId,
|
||||
ImportTaskIds: []string{
|
||||
*importTaskOutput.ImportTaskId,
|
||||
},
|
||||
},
|
||||
time.Hour*24,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
snapshotTaskStatus := *snapWaitOutput.ImportSnapshotTasks[0].SnapshotTaskDetail.Status
|
||||
if snapshotTaskStatus != "completed" {
|
||||
return nil, fmt.Errorf("Unable to import snapshot, task result: %v, msg: %v", snapshotTaskStatus, *snapWaitOutput.ImportSnapshotTasks[0].SnapshotTaskDetail.StatusMessage)
|
||||
}
|
||||
|
||||
// we no longer need the object in s3, let's just delete it
|
||||
logrus.Infof("[AWS] 🧹 Deleting image from S3: %s/%s", bucket, key)
|
||||
_, err = a.s3.DeleteObject(
|
||||
|
|
@ -347,24 +287,13 @@ func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch str
|
|||
return nil, err
|
||||
}
|
||||
|
||||
importOutput, err := a.ec2.DescribeImportSnapshotTasks(
|
||||
&ec2.DescribeImportSnapshotTasksInput{
|
||||
ImportTaskIds: []*string{
|
||||
importTaskOutput.ImportTaskId,
|
||||
},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
snapshotID := importOutput.ImportSnapshotTasks[0].SnapshotTaskDetail.SnapshotId
|
||||
|
||||
snapshotID := *snapWaitOutput.ImportSnapshotTasks[0].SnapshotTaskDetail.SnapshotId
|
||||
// Tag the snapshot with the image name.
|
||||
req, _ := a.ec2.CreateTagsRequest(
|
||||
_, err = a.ec2.CreateTags(
|
||||
context.Background(),
|
||||
&ec2.CreateTagsInput{
|
||||
Resources: []*string{snapshotID},
|
||||
Tags: []*ec2.Tag{
|
||||
Resources: []string{snapshotID},
|
||||
Tags: []ec2types.Tag{
|
||||
{
|
||||
Key: aws.String("Name"),
|
||||
Value: aws.String(name),
|
||||
|
|
@ -372,25 +301,25 @@ func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch str
|
|||
},
|
||||
},
|
||||
)
|
||||
err = req.Send()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
logrus.Infof("[AWS] 📋 Registering AMI from imported snapshot: %s", *snapshotID)
|
||||
logrus.Infof("[AWS] 📋 Registering AMI from imported snapshot: %s", snapshotID)
|
||||
registerOutput, err := a.ec2.RegisterImage(
|
||||
context.Background(),
|
||||
&ec2.RegisterImageInput{
|
||||
Architecture: aws.String(ec2Arch),
|
||||
BootMode: bootMode,
|
||||
Architecture: ec2Arch,
|
||||
BootMode: ec2BootMode,
|
||||
VirtualizationType: aws.String("hvm"),
|
||||
Name: aws.String(name),
|
||||
RootDeviceName: aws.String("/dev/sda1"),
|
||||
EnaSupport: aws.Bool(true),
|
||||
BlockDeviceMappings: []*ec2.BlockDeviceMapping{
|
||||
BlockDeviceMappings: []ec2types.BlockDeviceMapping{
|
||||
{
|
||||
DeviceName: aws.String("/dev/sda1"),
|
||||
Ebs: &ec2.EbsBlockDevice{
|
||||
SnapshotId: snapshotID,
|
||||
Ebs: &ec2types.EbsBlockDevice{
|
||||
SnapshotId: aws.String(snapshotID),
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -403,10 +332,11 @@ func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch str
|
|||
logrus.Infof("[AWS] 🎉 AMI registered: %s", *registerOutput.ImageId)
|
||||
|
||||
// Tag the image with the image name.
|
||||
req, _ = a.ec2.CreateTagsRequest(
|
||||
_, err = a.ec2.CreateTags(
|
||||
context.Background(),
|
||||
&ec2.CreateTagsInput{
|
||||
Resources: []*string{registerOutput.ImageId},
|
||||
Tags: []*ec2.Tag{
|
||||
Resources: []string{*registerOutput.ImageId},
|
||||
Tags: []ec2types.Tag{
|
||||
{
|
||||
Key: aws.String("Name"),
|
||||
Value: aws.String(name),
|
||||
|
|
@ -414,7 +344,6 @@ func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch str
|
|||
},
|
||||
},
|
||||
)
|
||||
err = req.Send()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -436,6 +365,7 @@ func (a *AWS) Register(name, bucket, key string, shareWith []string, rpmArch str
|
|||
// target region is determined by the region configured in the aws session
|
||||
func (a *AWS) CopyImage(name, ami, sourceRegion string) (string, error) {
|
||||
result, err := a.ec2.CopyImage(
|
||||
context.Background(),
|
||||
&ec2.CopyImageInput{
|
||||
Name: aws.String(name),
|
||||
SourceImageId: aws.String(ami),
|
||||
|
|
@ -446,61 +376,46 @@ func (a *AWS) CopyImage(name, ami, sourceRegion string) (string, error) {
|
|||
return "", err
|
||||
}
|
||||
|
||||
dIInput := &ec2.DescribeImagesInput{
|
||||
ImageIds: []*string{result.ImageId},
|
||||
}
|
||||
|
||||
// Custom waiter which waits indefinitely until a final state
|
||||
w := request.Waiter{
|
||||
Name: "WaitUntilImageAvailable",
|
||||
MaxAttempts: 0,
|
||||
Delay: request.ConstantWaiterDelay(15 * time.Second),
|
||||
Acceptors: []request.WaiterAcceptor{
|
||||
{
|
||||
State: request.SuccessWaiterState,
|
||||
Matcher: request.PathAllWaiterMatch, Argument: "Images[].State",
|
||||
Expected: "available",
|
||||
},
|
||||
{
|
||||
State: request.FailureWaiterState,
|
||||
Matcher: request.PathAnyWaiterMatch, Argument: "Images[].State",
|
||||
Expected: "failed",
|
||||
},
|
||||
imgWaiter := ec2.NewImageAvailableWaiter(a.ec2)
|
||||
imgWaitOutput, err := imgWaiter.WaitForOutput(
|
||||
context.Background(),
|
||||
&ec2.DescribeImagesInput{
|
||||
ImageIds: []string{*result.ImageId},
|
||||
},
|
||||
Logger: a.ec2.Config.Logger,
|
||||
NewRequest: func(opts []request.Option) (*request.Request, error) {
|
||||
var inCpy *ec2.DescribeImagesInput
|
||||
if dIInput != nil {
|
||||
tmp := *dIInput
|
||||
inCpy = &tmp
|
||||
}
|
||||
req, _ := a.ec2.DescribeImagesRequest(inCpy)
|
||||
req.SetContext(aws.BackgroundContext())
|
||||
req.ApplyOptions(opts...)
|
||||
return req, nil
|
||||
},
|
||||
}
|
||||
err = w.WaitWithContext(aws.BackgroundContext())
|
||||
time.Hour*24,
|
||||
)
|
||||
if err != nil {
|
||||
return *result.ImageId, err
|
||||
}
|
||||
|
||||
if imgWaitOutput.Images[0].State != ec2types.ImageStateAvailable {
|
||||
return *result.ImageId, fmt.Errorf("Image not available after waiting: %s, Code: %v reason: %v",
|
||||
imgWaitOutput.Images[0].State, *imgWaitOutput.Images[0].StateReason.Code, *imgWaitOutput.Images[0].StateReason.Message)
|
||||
}
|
||||
|
||||
// Tag image with name
|
||||
_, err = a.ec2.CreateTags(&ec2.CreateTagsInput{
|
||||
Resources: []*string{result.ImageId},
|
||||
Tags: []*ec2.Tag{
|
||||
{
|
||||
Key: aws.String("Name"),
|
||||
Value: aws.String(name),
|
||||
_, err = a.ec2.CreateTags(
|
||||
context.Background(),
|
||||
&ec2.CreateTagsInput{
|
||||
Resources: []string{*result.ImageId},
|
||||
Tags: []ec2types.Tag{
|
||||
{
|
||||
Key: aws.String("Name"),
|
||||
Value: aws.String(name),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
)
|
||||
if err != nil {
|
||||
return *result.ImageId, err
|
||||
}
|
||||
|
||||
imgs, err := a.ec2.DescribeImages(dIInput)
|
||||
imgs, err := a.ec2.DescribeImages(
|
||||
context.Background(),
|
||||
&ec2.DescribeImagesInput{
|
||||
ImageIds: []string{*result.ImageId},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return *result.ImageId, err
|
||||
}
|
||||
|
|
@ -510,15 +425,17 @@ func (a *AWS) CopyImage(name, ami, sourceRegion string) (string, error) {
|
|||
|
||||
// Tag snapshot with name
|
||||
for _, bdm := range imgs.Images[0].BlockDeviceMappings {
|
||||
_, err = a.ec2.CreateTags(&ec2.CreateTagsInput{
|
||||
Resources: []*string{bdm.Ebs.SnapshotId},
|
||||
Tags: []*ec2.Tag{
|
||||
{
|
||||
Key: aws.String("Name"),
|
||||
Value: aws.String(name),
|
||||
_, err = a.ec2.CreateTags(
|
||||
context.Background(),
|
||||
&ec2.CreateTagsInput{
|
||||
Resources: []string{*bdm.Ebs.SnapshotId},
|
||||
Tags: []ec2types.Tag{
|
||||
{
|
||||
Key: aws.String("Name"),
|
||||
Value: aws.String(name),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return *result.ImageId, err
|
||||
}
|
||||
|
|
@ -529,8 +446,9 @@ func (a *AWS) CopyImage(name, ami, sourceRegion string) (string, error) {
|
|||
|
||||
func (a *AWS) ShareImage(ami string, userIds []string) error {
|
||||
imgs, err := a.ec2.DescribeImages(
|
||||
context.Background(),
|
||||
&ec2.DescribeImagesInput{
|
||||
ImageIds: []*string{aws.String(ami)},
|
||||
ImageIds: []string{ami},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
|
|
@ -541,7 +459,7 @@ func (a *AWS) ShareImage(ami string, userIds []string) error {
|
|||
}
|
||||
|
||||
for _, bdm := range imgs.Images[0].BlockDeviceMappings {
|
||||
err = a.shareSnapshot(bdm.Ebs.SnapshotId, userIds)
|
||||
err = a.shareSnapshot(*bdm.Ebs.SnapshotId, userIds)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -562,16 +480,17 @@ func (a *AWS) shareImage(ami *string, userIds []string) error {
|
|||
}
|
||||
|
||||
logrus.Info("[AWS] 💿 Sharing ec2 AMI")
|
||||
var launchPerms []*ec2.LaunchPermission
|
||||
var launchPerms []ec2types.LaunchPermission
|
||||
for _, id := range uIds {
|
||||
launchPerms = append(launchPerms, &ec2.LaunchPermission{
|
||||
launchPerms = append(launchPerms, ec2types.LaunchPermission{
|
||||
UserId: id,
|
||||
})
|
||||
}
|
||||
_, err := a.ec2.ModifyImageAttribute(
|
||||
context.Background(),
|
||||
&ec2.ModifyImageAttributeInput{
|
||||
ImageId: ami,
|
||||
LaunchPermission: &ec2.LaunchPermissionModifications{
|
||||
LaunchPermission: &ec2types.LaunchPermissionModifications{
|
||||
Add: launchPerms,
|
||||
},
|
||||
},
|
||||
|
|
@ -584,18 +503,15 @@ func (a *AWS) shareImage(ami *string, userIds []string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (a *AWS) shareSnapshot(snapshotId *string, userIds []string) error {
|
||||
func (a *AWS) shareSnapshot(snapshotId string, userIds []string) error {
|
||||
logrus.Info("[AWS] 🎥 Sharing ec2 snapshot")
|
||||
var uIds []*string
|
||||
for i := range userIds {
|
||||
uIds = append(uIds, &userIds[i])
|
||||
}
|
||||
_, err := a.ec2.ModifySnapshotAttribute(
|
||||
context.Background(),
|
||||
&ec2.ModifySnapshotAttributeInput{
|
||||
Attribute: aws.String(ec2.SnapshotAttributeNameCreateVolumePermission),
|
||||
OperationType: aws.String("add"),
|
||||
SnapshotId: snapshotId,
|
||||
UserIds: uIds,
|
||||
Attribute: ec2types.SnapshotAttributeNameCreateVolumePermission,
|
||||
OperationType: ec2types.OperationTypeAdd,
|
||||
SnapshotId: aws.String(snapshotId),
|
||||
UserIds: userIds,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
|
|
@ -606,7 +522,7 @@ func (a *AWS) shareSnapshot(snapshotId *string, userIds []string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (a *AWS) RemoveSnapshotAndDeregisterImage(image *ec2.Image) error {
|
||||
func (a *AWS) RemoveSnapshotAndDeregisterImage(image *ec2types.Image) error {
|
||||
if image == nil {
|
||||
return fmt.Errorf("image is nil")
|
||||
}
|
||||
|
|
@ -617,6 +533,7 @@ func (a *AWS) RemoveSnapshotAndDeregisterImage(image *ec2.Image) error {
|
|||
}
|
||||
|
||||
_, err := a.ec2.DeregisterImage(
|
||||
context.Background(),
|
||||
&ec2.DeregisterImageInput{
|
||||
ImageId: image.ImageId,
|
||||
},
|
||||
|
|
@ -627,6 +544,7 @@ func (a *AWS) RemoveSnapshotAndDeregisterImage(image *ec2.Image) error {
|
|||
|
||||
for _, s := range snapshots {
|
||||
_, err = a.ec2.DeleteSnapshot(
|
||||
context.Background(),
|
||||
&ec2.DeleteSnapshotInput{
|
||||
SnapshotId: s,
|
||||
},
|
||||
|
|
@ -642,13 +560,14 @@ func (a *AWS) RemoveSnapshotAndDeregisterImage(image *ec2.Image) error {
|
|||
// For service maintenance images are discovered by the "Name:composer-api-*" tag filter. Currently
|
||||
// all image names in the service are generated, so they're guaranteed to be unique as well. If
|
||||
// users are ever allowed to name their images, an extra tag should be added.
|
||||
func (a *AWS) DescribeImagesByTag(tagKey, tagValue string) ([]*ec2.Image, error) {
|
||||
func (a *AWS) DescribeImagesByTag(tagKey, tagValue string) ([]ec2types.Image, error) {
|
||||
imgs, err := a.ec2.DescribeImages(
|
||||
context.Background(),
|
||||
&ec2.DescribeImagesInput{
|
||||
Filters: []*ec2.Filter{
|
||||
Filters: []ec2types.Filter{
|
||||
{
|
||||
Name: aws.String(fmt.Sprintf("tag:%s", tagKey)),
|
||||
Values: []*string{aws.String(tagValue)},
|
||||
Values: []string{tagValue},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
@ -695,14 +614,14 @@ func (a *AWS) MarkS3ObjectAsPublic(bucket, objectKey string) error {
|
|||
}
|
||||
|
||||
func (a *AWS) Regions() ([]string, error) {
|
||||
out, err := a.ec2.DescribeRegions(&ec2.DescribeRegionsInput{})
|
||||
out, err := a.ec2.DescribeRegions(context.Background(), &ec2.DescribeRegionsInput{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := []string{}
|
||||
for _, r := range out.Regions {
|
||||
result = append(result, aws.StringValue(r.RegionName))
|
||||
result = append(result, *r.RegionName)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,10 +4,61 @@ import (
|
|||
"context"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
|
||||
"github.com/aws/aws-sdk-go-v2/service/ec2"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
)
|
||||
|
||||
type EC2 interface {
|
||||
DescribeRegions(context.Context, *ec2.DescribeRegionsInput, ...func(*ec2.Options)) (*ec2.DescribeRegionsOutput, error)
|
||||
|
||||
// Security Groups
|
||||
AuthorizeSecurityGroupIngress(context.Context, *ec2.AuthorizeSecurityGroupIngressInput, ...func(*ec2.Options)) (*ec2.AuthorizeSecurityGroupIngressOutput, error)
|
||||
CreateSecurityGroup(context.Context, *ec2.CreateSecurityGroupInput, ...func(*ec2.Options)) (*ec2.CreateSecurityGroupOutput, error)
|
||||
DeleteSecurityGroup(context.Context, *ec2.DeleteSecurityGroupInput, ...func(*ec2.Options)) (*ec2.DeleteSecurityGroupOutput, error)
|
||||
DescribeSecurityGroups(context.Context, *ec2.DescribeSecurityGroupsInput, ...func(*ec2.Options)) (*ec2.DescribeSecurityGroupsOutput, error)
|
||||
|
||||
// Subnets
|
||||
CreateSubnet(context.Context, *ec2.CreateSubnetInput, ...func(*ec2.Options)) (*ec2.CreateSubnetOutput, error)
|
||||
DeleteSubnet(context.Context, *ec2.DeleteSubnetInput, ...func(*ec2.Options)) (*ec2.DeleteSubnetOutput, error)
|
||||
DescribeSubnets(context.Context, *ec2.DescribeSubnetsInput, ...func(*ec2.Options)) (*ec2.DescribeSubnetsOutput, error)
|
||||
|
||||
// LaunchTemplates
|
||||
CreateLaunchTemplate(context.Context, *ec2.CreateLaunchTemplateInput, ...func(*ec2.Options)) (*ec2.CreateLaunchTemplateOutput, error)
|
||||
DeleteLaunchTemplate(context.Context, *ec2.DeleteLaunchTemplateInput, ...func(*ec2.Options)) (*ec2.DeleteLaunchTemplateOutput, error)
|
||||
DescribeLaunchTemplates(context.Context, *ec2.DescribeLaunchTemplatesInput, ...func(*ec2.Options)) (*ec2.DescribeLaunchTemplatesOutput, error)
|
||||
|
||||
// Instances
|
||||
DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error)
|
||||
DescribeInstanceStatus(context.Context, *ec2.DescribeInstanceStatusInput, ...func(*ec2.Options)) (*ec2.DescribeInstanceStatusOutput, error)
|
||||
TerminateInstances(context.Context, *ec2.TerminateInstancesInput, ...func(*ec2.Options)) (*ec2.TerminateInstancesOutput, error)
|
||||
|
||||
// Fleets
|
||||
CreateFleet(context.Context, *ec2.CreateFleetInput, ...func(*ec2.Options)) (*ec2.CreateFleetOutput, error)
|
||||
DeleteFleets(context.Context, *ec2.DeleteFleetsInput, ...func(*ec2.Options)) (*ec2.DeleteFleetsOutput, error)
|
||||
|
||||
// Images
|
||||
CopyImage(context.Context, *ec2.CopyImageInput, ...func(*ec2.Options)) (*ec2.CopyImageOutput, error)
|
||||
RegisterImage(context.Context, *ec2.RegisterImageInput, ...func(*ec2.Options)) (*ec2.RegisterImageOutput, error)
|
||||
DeregisterImage(context.Context, *ec2.DeregisterImageInput, ...func(*ec2.Options)) (*ec2.DeregisterImageOutput, error)
|
||||
DescribeImages(context.Context, *ec2.DescribeImagesInput, ...func(*ec2.Options)) (*ec2.DescribeImagesOutput, error)
|
||||
ModifyImageAttribute(context.Context, *ec2.ModifyImageAttributeInput, ...func(*ec2.Options)) (*ec2.ModifyImageAttributeOutput, error)
|
||||
|
||||
// Snapshots
|
||||
DeleteSnapshot(context.Context, *ec2.DeleteSnapshotInput, ...func(*ec2.Options)) (*ec2.DeleteSnapshotOutput, error)
|
||||
DescribeImportSnapshotTasks(context.Context, *ec2.DescribeImportSnapshotTasksInput, ...func(*ec2.Options)) (*ec2.DescribeImportSnapshotTasksOutput, error)
|
||||
ImportSnapshot(context.Context, *ec2.ImportSnapshotInput, ...func(*ec2.Options)) (*ec2.ImportSnapshotOutput, error)
|
||||
ModifySnapshotAttribute(context.Context, *ec2.ModifySnapshotAttributeInput, ...func(*ec2.Options)) (*ec2.ModifySnapshotAttributeOutput, error)
|
||||
|
||||
// Tags
|
||||
CreateTags(context.Context, *ec2.CreateTagsInput, ...func(*ec2.Options)) (*ec2.CreateTagsOutput, error)
|
||||
}
|
||||
|
||||
type EC2Imds interface {
|
||||
GetInstanceIdentityDocument(context.Context, *imds.GetInstanceIdentityDocumentInput, ...func(*imds.Options)) (*imds.GetInstanceIdentityDocumentOutput, error)
|
||||
}
|
||||
|
||||
type S3 interface {
|
||||
DeleteObject(context.Context, *s3.DeleteObjectInput, ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
|
||||
PutObjectAcl(context.Context, *s3.PutObjectAclInput, ...func(*s3.Options)) (*s3.PutObjectAclOutput, error)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,18 @@
|
|||
package awscloud
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/service/ec2"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
|
||||
"github.com/aws/aws-sdk-go-v2/service/ec2"
|
||||
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
|
||||
smithy "github.com/aws/smithy-go"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
|
|
@ -15,7 +20,7 @@ type SecureInstance struct {
|
|||
FleetID string
|
||||
SGID string
|
||||
LTID string
|
||||
Instance *ec2.Instance
|
||||
Instance *ec2types.Instance
|
||||
InstanceID string
|
||||
}
|
||||
|
||||
|
|
@ -47,17 +52,20 @@ write_files:
|
|||
// Runs an instance with a security group that only allows traffic to
|
||||
// the host. Will replace resources if they already exists.
|
||||
func (a *AWS) RunSecureInstance(iamProfile, keyName, cloudWatchGroup, hostname string) (*SecureInstance, error) {
|
||||
identity, err := a.ec2metadata.GetInstanceIdentityDocument()
|
||||
identity, err := a.ec2imds.GetInstanceIdentityDocument(context.Background(), &imds.GetInstanceIdentityDocumentInput{})
|
||||
if err != nil {
|
||||
logrus.Errorf("Error getting the identity document, %s", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
descrInstancesOutput, err := a.ec2.DescribeInstances(&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []*string{
|
||||
aws.String(identity.InstanceID),
|
||||
descrInstancesOutput, err := a.ec2.DescribeInstances(
|
||||
context.Background(),
|
||||
&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []string{
|
||||
identity.InstanceID,
|
||||
},
|
||||
},
|
||||
})
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -66,7 +74,6 @@ func (a *AWS) RunSecureInstance(iamProfile, keyName, cloudWatchGroup, hostname s
|
|||
}
|
||||
vpcID := *descrInstancesOutput.Reservations[0].Instances[0].VpcId
|
||||
imageID := *descrInstancesOutput.Reservations[0].Instances[0].ImageId
|
||||
instanceType := *descrInstancesOutput.Reservations[0].Instances[0].InstanceType
|
||||
subnetID := *descrInstancesOutput.Reservations[0].Instances[0].SubnetId
|
||||
|
||||
secureInstance := &SecureInstance{}
|
||||
|
|
@ -96,7 +103,7 @@ func (a *AWS) RunSecureInstance(iamProfile, keyName, cloudWatchGroup, hostname s
|
|||
return nil, err
|
||||
}
|
||||
|
||||
ltID, err := a.createOrReplaceLT(identity.InstanceID, imageID, sgID, instanceType, iamProfile, keyName, cloudWatchGroup, hostname)
|
||||
ltID, err := a.createOrReplaceLT(identity.InstanceID, imageID, sgID, iamProfile, keyName, cloudWatchGroup, hostname)
|
||||
if ltID != "" {
|
||||
secureInstance.LTID = ltID
|
||||
}
|
||||
|
|
@ -104,16 +111,18 @@ func (a *AWS) RunSecureInstance(iamProfile, keyName, cloudWatchGroup, hostname s
|
|||
return nil, err
|
||||
}
|
||||
|
||||
descrSubnetsOutput, err := a.ec2.DescribeSubnets(&ec2.DescribeSubnetsInput{
|
||||
Filters: []*ec2.Filter{
|
||||
&ec2.Filter{
|
||||
Name: aws.String("vpc-id"),
|
||||
Values: []*string{
|
||||
aws.String(vpcID),
|
||||
descrSubnetsOutput, err := a.ec2.DescribeSubnets(
|
||||
context.Background(),
|
||||
&ec2.DescribeSubnetsInput{
|
||||
Filters: []ec2types.Filter{
|
||||
ec2types.Filter{
|
||||
Name: aws.String("vpc-id"),
|
||||
Values: []string{
|
||||
vpcID,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -122,59 +131,66 @@ func (a *AWS) RunSecureInstance(iamProfile, keyName, cloudWatchGroup, hostname s
|
|||
}
|
||||
|
||||
createFleetOutput, err := a.createFleet(&ec2.CreateFleetInput{
|
||||
LaunchTemplateConfigs: []*ec2.FleetLaunchTemplateConfigRequest{
|
||||
&ec2.FleetLaunchTemplateConfigRequest{
|
||||
LaunchTemplateSpecification: &ec2.FleetLaunchTemplateSpecificationRequest{
|
||||
LaunchTemplateConfigs: []ec2types.FleetLaunchTemplateConfigRequest{
|
||||
ec2types.FleetLaunchTemplateConfigRequest{
|
||||
LaunchTemplateSpecification: &ec2types.FleetLaunchTemplateSpecificationRequest{
|
||||
LaunchTemplateId: aws.String(secureInstance.LTID),
|
||||
Version: aws.String("1"),
|
||||
},
|
||||
Overrides: []*ec2.FleetLaunchTemplateOverridesRequest{
|
||||
&ec2.FleetLaunchTemplateOverridesRequest{
|
||||
Overrides: []ec2types.FleetLaunchTemplateOverridesRequest{
|
||||
ec2types.FleetLaunchTemplateOverridesRequest{
|
||||
SubnetId: aws.String(subnetID),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
TagSpecifications: []*ec2.TagSpecification{
|
||||
&ec2.TagSpecification{
|
||||
ResourceType: aws.String(ec2.ResourceTypeInstance),
|
||||
Tags: []*ec2.Tag{
|
||||
&ec2.Tag{
|
||||
TagSpecifications: []ec2types.TagSpecification{
|
||||
ec2types.TagSpecification{
|
||||
ResourceType: ec2types.ResourceTypeInstance,
|
||||
Tags: []ec2types.Tag{
|
||||
ec2types.Tag{
|
||||
Key: aws.String("parent"),
|
||||
Value: aws.String(identity.InstanceID),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
TargetCapacitySpecification: &ec2.TargetCapacitySpecificationRequest{
|
||||
DefaultTargetCapacityType: aws.String(ec2.DefaultTargetCapacityTypeSpot),
|
||||
TotalTargetCapacity: aws.Int64(1),
|
||||
TargetCapacitySpecification: &ec2types.TargetCapacitySpecificationRequest{
|
||||
DefaultTargetCapacityType: ec2types.DefaultTargetCapacityTypeSpot,
|
||||
TotalTargetCapacity: aws.Int32(1),
|
||||
},
|
||||
SpotOptions: &ec2.SpotOptionsRequest{
|
||||
AllocationStrategy: aws.String(ec2.SpotAllocationStrategyPriceCapacityOptimized),
|
||||
SpotOptions: &ec2types.SpotOptionsRequest{
|
||||
AllocationStrategy: ec2types.SpotAllocationStrategyPriceCapacityOptimized,
|
||||
},
|
||||
Type: aws.String(ec2.FleetTypeInstant),
|
||||
Type: ec2types.FleetTypeInstant,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
secureInstance.FleetID = *createFleetOutput.FleetId
|
||||
secureInstance.InstanceID = createFleetOutput.Instances[0].InstanceIds[0]
|
||||
|
||||
secureInstance.InstanceID = *createFleetOutput.Instances[0].InstanceIds[0]
|
||||
err = a.ec2.WaitUntilInstanceStatusOk(&ec2.DescribeInstanceStatusInput{
|
||||
InstanceIds: []*string{
|
||||
aws.String(secureInstance.InstanceID),
|
||||
instWaiter := ec2.NewInstanceStatusOkWaiter(a.ec2)
|
||||
err = instWaiter.Wait(
|
||||
context.Background(),
|
||||
&ec2.DescribeInstanceStatusInput{
|
||||
InstanceIds: []string{
|
||||
secureInstance.InstanceID,
|
||||
},
|
||||
},
|
||||
})
|
||||
time.Hour,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
descrInstOutput, err := a.ec2.DescribeInstances(&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []*string{
|
||||
aws.String(secureInstance.InstanceID),
|
||||
},
|
||||
})
|
||||
descrInstOutput, err := a.ec2.DescribeInstances(
|
||||
context.Background(),
|
||||
&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []string{
|
||||
secureInstance.InstanceID,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -184,7 +200,7 @@ func (a *AWS) RunSecureInstance(iamProfile, keyName, cloudWatchGroup, hostname s
|
|||
if len(descrInstOutput.Reservations[0].Instances) != 1 {
|
||||
return nil, fmt.Errorf("Expected exactly 1 instance for instance: %s, got %d", secureInstance.InstanceID, len(descrInstOutput.Reservations[0].Instances))
|
||||
}
|
||||
secureInstance.Instance = descrInstOutput.Reservations[0].Instances[0]
|
||||
secureInstance.Instance = &descrInstOutput.Reservations[0].Instances[0]
|
||||
|
||||
return secureInstance, nil
|
||||
}
|
||||
|
|
@ -205,14 +221,16 @@ func (a *AWS) TerminateSecureInstance(si *SecureInstance) error {
|
|||
}
|
||||
|
||||
func (a *AWS) terminatePreviousSI(hostInstanceID string) (string, error) {
|
||||
descrInstancesOutput, err := a.ec2.DescribeInstances(&ec2.DescribeInstancesInput{
|
||||
Filters: []*ec2.Filter{
|
||||
&ec2.Filter{
|
||||
Name: aws.String("tag:parent"),
|
||||
Values: []*string{aws.String(hostInstanceID)},
|
||||
descrInstancesOutput, err := a.ec2.DescribeInstances(
|
||||
context.Background(),
|
||||
&ec2.DescribeInstancesInput{
|
||||
Filters: []ec2types.Filter{
|
||||
ec2types.Filter{
|
||||
Name: aws.String("tag:parent"),
|
||||
Values: []string{hostInstanceID},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
@ -220,29 +238,39 @@ func (a *AWS) terminatePreviousSI(hostInstanceID string) (string, error) {
|
|||
return "", nil
|
||||
}
|
||||
|
||||
if *descrInstancesOutput.Reservations[0].Instances[0].State.Name == ec2.InstanceStateNameTerminated {
|
||||
if descrInstancesOutput.Reservations[0].Instances[0].State.Name == ec2types.InstanceStateNameTerminated {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
instanceID := descrInstancesOutput.Reservations[0].Instances[0].InstanceId
|
||||
_, err = a.ec2.TerminateInstances(&ec2.TerminateInstancesInput{
|
||||
InstanceIds: []*string{instanceID},
|
||||
})
|
||||
instanceID := *descrInstancesOutput.Reservations[0].Instances[0].InstanceId
|
||||
_, err = a.ec2.TerminateInstances(
|
||||
context.Background(),
|
||||
&ec2.TerminateInstancesInput{
|
||||
InstanceIds: []string{instanceID},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return *instanceID, err
|
||||
return instanceID, err
|
||||
}
|
||||
err = a.ec2.WaitUntilInstanceTerminated(&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []*string{instanceID},
|
||||
})
|
||||
|
||||
instTermWaiter := ec2.NewInstanceTerminatedWaiter(a.ec2)
|
||||
err = instTermWaiter.Wait(
|
||||
context.Background(),
|
||||
&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []string{instanceID},
|
||||
},
|
||||
time.Hour,
|
||||
)
|
||||
if err != nil {
|
||||
return *instanceID, err
|
||||
return instanceID, err
|
||||
}
|
||||
return *instanceID, nil
|
||||
return instanceID, nil
|
||||
}
|
||||
|
||||
func isInvalidGroupNotFoundErr(err error) bool {
|
||||
if awsErr, ok := err.(awserr.Error); ok {
|
||||
if awsErr.Code() == "InvalidGroup.NotFound" {
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
if apiErr.ErrorCode() == "InvalidGroup.NotFound" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
|
@ -251,53 +279,63 @@ func isInvalidGroupNotFoundErr(err error) bool {
|
|||
|
||||
func (a *AWS) createOrReplaceSG(hostInstanceID, hostIP, vpcID string) (string, error) {
|
||||
sgName := fmt.Sprintf("SG for %s (%s)", hostInstanceID, hostIP)
|
||||
descrSGOutput, err := a.ec2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
|
||||
Filters: []*ec2.Filter{
|
||||
&ec2.Filter{
|
||||
Name: aws.String("group-name"),
|
||||
Values: []*string{
|
||||
aws.String(sgName),
|
||||
descrSGOutput, err := a.ec2.DescribeSecurityGroups(
|
||||
context.Background(),
|
||||
&ec2.DescribeSecurityGroupsInput{
|
||||
Filters: []ec2types.Filter{
|
||||
ec2types.Filter{
|
||||
Name: aws.String("group-name"),
|
||||
Values: []string{
|
||||
sgName,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
if err != nil && !isInvalidGroupNotFoundErr(err) {
|
||||
return "", err
|
||||
}
|
||||
for _, sg := range descrSGOutput.SecurityGroups {
|
||||
_, err := a.ec2.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
|
||||
GroupId: sg.GroupId,
|
||||
})
|
||||
_, err := a.ec2.DeleteSecurityGroup(
|
||||
context.Background(),
|
||||
&ec2.DeleteSecurityGroupInput{
|
||||
GroupId: sg.GroupId,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
cSGOutput, err := a.ec2.CreateSecurityGroup(&ec2.CreateSecurityGroupInput{
|
||||
Description: aws.String(sgName),
|
||||
GroupName: aws.String(sgName),
|
||||
VpcId: aws.String(vpcID),
|
||||
})
|
||||
cSGOutput, err := a.ec2.CreateSecurityGroup(
|
||||
context.Background(),
|
||||
&ec2.CreateSecurityGroupInput{
|
||||
Description: aws.String(sgName),
|
||||
GroupName: aws.String(sgName),
|
||||
VpcId: aws.String(vpcID),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sgID := *cSGOutput.GroupId
|
||||
|
||||
sgIngressOutput, err := a.ec2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{
|
||||
GroupId: aws.String(sgID),
|
||||
IpPermissions: []*ec2.IpPermission{
|
||||
&ec2.IpPermission{
|
||||
IpProtocol: aws.String(ec2.ProtocolTcp),
|
||||
FromPort: aws.Int64(1),
|
||||
ToPort: aws.Int64(65535),
|
||||
IpRanges: []*ec2.IpRange{
|
||||
&ec2.IpRange{
|
||||
CidrIp: aws.String(fmt.Sprintf("%s/32", hostIP)),
|
||||
sgIngressOutput, err := a.ec2.AuthorizeSecurityGroupIngress(
|
||||
context.Background(),
|
||||
&ec2.AuthorizeSecurityGroupIngressInput{
|
||||
GroupId: aws.String(sgID),
|
||||
IpPermissions: []ec2types.IpPermission{
|
||||
ec2types.IpPermission{
|
||||
IpProtocol: aws.String(string(ec2types.ProtocolTcp)),
|
||||
FromPort: aws.Int32(1),
|
||||
ToPort: aws.Int32(65535),
|
||||
IpRanges: []ec2types.IpRange{
|
||||
ec2types.IpRange{
|
||||
CidrIp: aws.String(fmt.Sprintf("%s/32", hostIP)),
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
})
|
||||
if err != nil {
|
||||
return sgID, err
|
||||
}
|
||||
|
|
@ -305,11 +343,14 @@ func (a *AWS) createOrReplaceSG(hostInstanceID, hostIP, vpcID string) (string, e
|
|||
return sgID, fmt.Errorf("Unable to attach ingress rules to SG")
|
||||
}
|
||||
|
||||
describeSGOutput, err := a.ec2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{
|
||||
GroupIds: []*string{
|
||||
aws.String(sgID),
|
||||
describeSGOutput, err := a.ec2.DescribeSecurityGroups(
|
||||
context.Background(),
|
||||
&ec2.DescribeSecurityGroupsInput{
|
||||
GroupIds: []string{
|
||||
sgID,
|
||||
},
|
||||
},
|
||||
})
|
||||
)
|
||||
if err != nil {
|
||||
return sgID, err
|
||||
}
|
||||
|
|
@ -324,26 +365,32 @@ func (a *AWS) createOrReplaceSG(hostInstanceID, hostIP, vpcID string) (string, e
|
|||
}
|
||||
|
||||
func isLaunchTemplateNotFoundError(err error) bool {
|
||||
if awsErr, ok := err.(awserr.Error); ok {
|
||||
if awsErr.Code() == "InvalidLaunchTemplateId.NotFound" || awsErr.Code() == "InvalidLaunchTemplateName.NotFoundException" {
|
||||
var apiErr smithy.APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
if apiErr.ErrorCode() == "InvalidLaunchTemplateId.NotFound" || apiErr.ErrorCode() == "InvalidLaunchTemplateName.NotFoundException" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
||||
}
|
||||
|
||||
func (a *AWS) createOrReplaceLT(hostInstanceID, imageID, sgID, instanceType, iamProfile, keyName, cloudWatchGroup, hostname string) (string, error) {
|
||||
func (a *AWS) createOrReplaceLT(hostInstanceID, imageID, sgID, iamProfile, keyName, cloudWatchGroup, hostname string) (string, error) {
|
||||
ltName := fmt.Sprintf("launch-template-for-%s-runner-instance", hostInstanceID)
|
||||
descrLTOutput, err := a.ec2.DescribeLaunchTemplates(&ec2.DescribeLaunchTemplatesInput{
|
||||
LaunchTemplateNames: []*string{
|
||||
aws.String(ltName),
|
||||
descrLTOutput, err := a.ec2.DescribeLaunchTemplates(
|
||||
context.Background(),
|
||||
&ec2.DescribeLaunchTemplatesInput{
|
||||
LaunchTemplateNames: []string{
|
||||
ltName,
|
||||
},
|
||||
},
|
||||
})
|
||||
)
|
||||
if len(descrLTOutput.LaunchTemplates) == 1 {
|
||||
_, err := a.ec2.DeleteLaunchTemplate(&ec2.DeleteLaunchTemplateInput{
|
||||
LaunchTemplateId: descrLTOutput.LaunchTemplates[0].LaunchTemplateId,
|
||||
})
|
||||
_, err := a.ec2.DeleteLaunchTemplate(
|
||||
context.Background(),
|
||||
&ec2.DeleteLaunchTemplateInput{
|
||||
LaunchTemplateId: descrLTOutput.LaunchTemplates[0].LaunchTemplateId,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
@ -353,46 +400,46 @@ func (a *AWS) createOrReplaceLT(hostInstanceID, imageID, sgID, instanceType, iam
|
|||
}
|
||||
|
||||
input := &ec2.CreateLaunchTemplateInput{
|
||||
LaunchTemplateData: &ec2.RequestLaunchTemplateData{
|
||||
LaunchTemplateData: &ec2types.RequestLaunchTemplateData{
|
||||
ImageId: aws.String(imageID),
|
||||
InstanceInitiatedShutdownBehavior: aws.String(ec2.ShutdownBehaviorTerminate),
|
||||
InstanceRequirements: &ec2.InstanceRequirementsRequest{
|
||||
AcceleratorCount: &ec2.AcceleratorCountRequest{
|
||||
Max: aws.Int64(0),
|
||||
InstanceInitiatedShutdownBehavior: ec2types.ShutdownBehaviorTerminate,
|
||||
InstanceRequirements: &ec2types.InstanceRequirementsRequest{
|
||||
AcceleratorCount: &ec2types.AcceleratorCountRequest{
|
||||
Max: aws.Int32(0),
|
||||
},
|
||||
BareMetal: aws.String(ec2.BareMetalExcluded),
|
||||
MemoryMiB: &ec2.MemoryMiBRequest{
|
||||
Min: aws.Int64(4096),
|
||||
BareMetal: ec2types.BareMetalExcluded,
|
||||
MemoryMiB: &ec2types.MemoryMiBRequest{
|
||||
Min: aws.Int32(4096),
|
||||
},
|
||||
NetworkInterfaceCount: &ec2.NetworkInterfaceCountRequest{
|
||||
Min: aws.Int64(1),
|
||||
NetworkInterfaceCount: &ec2types.NetworkInterfaceCountRequest{
|
||||
Min: aws.Int32(1),
|
||||
},
|
||||
SpotMaxPricePercentageOverLowestPrice: aws.Int64(200),
|
||||
VCpuCount: &ec2.VCpuCountRangeRequest{
|
||||
Min: aws.Int64(2),
|
||||
SpotMaxPricePercentageOverLowestPrice: aws.Int32(200),
|
||||
VCpuCount: &ec2types.VCpuCountRangeRequest{
|
||||
Min: aws.Int32(2),
|
||||
},
|
||||
},
|
||||
BlockDeviceMappings: []*ec2.LaunchTemplateBlockDeviceMappingRequest{
|
||||
&ec2.LaunchTemplateBlockDeviceMappingRequest{
|
||||
BlockDeviceMappings: []ec2types.LaunchTemplateBlockDeviceMappingRequest{
|
||||
ec2types.LaunchTemplateBlockDeviceMappingRequest{
|
||||
DeviceName: aws.String("/dev/sda1"),
|
||||
Ebs: &ec2.LaunchTemplateEbsBlockDeviceRequest{
|
||||
Ebs: &ec2types.LaunchTemplateEbsBlockDeviceRequest{
|
||||
DeleteOnTermination: aws.Bool(true),
|
||||
Encrypted: aws.Bool(true),
|
||||
VolumeSize: aws.Int64(50),
|
||||
VolumeType: aws.String(ec2.VolumeTypeGp3),
|
||||
VolumeSize: aws.Int32(50),
|
||||
VolumeType: ec2types.VolumeTypeGp3,
|
||||
},
|
||||
},
|
||||
},
|
||||
SecurityGroupIds: []*string{
|
||||
aws.String(sgID),
|
||||
SecurityGroupIds: []string{
|
||||
sgID,
|
||||
},
|
||||
UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(SecureInstanceUserData(cloudWatchGroup, hostname)))),
|
||||
},
|
||||
TagSpecifications: []*ec2.TagSpecification{
|
||||
&ec2.TagSpecification{
|
||||
ResourceType: aws.String(ec2.ResourceTypeLaunchTemplate),
|
||||
Tags: []*ec2.Tag{
|
||||
&ec2.Tag{
|
||||
TagSpecifications: []ec2types.TagSpecification{
|
||||
ec2types.TagSpecification{
|
||||
ResourceType: ec2types.ResourceTypeLaunchTemplate,
|
||||
Tags: []ec2types.Tag{
|
||||
ec2types.Tag{
|
||||
Key: aws.String("parent"),
|
||||
Value: aws.String(hostInstanceID),
|
||||
},
|
||||
|
|
@ -403,7 +450,7 @@ func (a *AWS) createOrReplaceLT(hostInstanceID, imageID, sgID, instanceType, iam
|
|||
}
|
||||
|
||||
if iamProfile != "" {
|
||||
input.LaunchTemplateData.IamInstanceProfile = &ec2.LaunchTemplateIamInstanceProfileSpecificationRequest{
|
||||
input.LaunchTemplateData.IamInstanceProfile = &ec2types.LaunchTemplateIamInstanceProfileSpecificationRequest{
|
||||
Name: aws.String(iamProfile),
|
||||
}
|
||||
}
|
||||
|
|
@ -412,7 +459,7 @@ func (a *AWS) createOrReplaceLT(hostInstanceID, imageID, sgID, instanceType, iam
|
|||
input.LaunchTemplateData.KeyName = aws.String(keyName)
|
||||
}
|
||||
|
||||
createLaunchTemplateOutput, err := a.ec2.CreateLaunchTemplate(input)
|
||||
createLaunchTemplateOutput, err := a.ec2.CreateLaunchTemplate(context.Background(), input)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
@ -424,12 +471,14 @@ func (a *AWS) deleteFleetIfExists(si *SecureInstance) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
delFlOutput, err := a.ec2.DeleteFleets(&ec2.DeleteFleetsInput{
|
||||
FleetIds: []*string{
|
||||
aws.String(si.FleetID),
|
||||
},
|
||||
TerminateInstances: aws.Bool(true),
|
||||
})
|
||||
delFlOutput, err := a.ec2.DeleteFleets(
|
||||
context.Background(),
|
||||
&ec2.DeleteFleetsInput{
|
||||
FleetIds: []string{
|
||||
si.FleetID,
|
||||
},
|
||||
TerminateInstances: aws.Bool(true),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -438,11 +487,14 @@ func (a *AWS) deleteFleetIfExists(si *SecureInstance) error {
|
|||
}
|
||||
|
||||
if si.InstanceID != "" {
|
||||
err = a.ec2.WaitUntilInstanceTerminated(&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []*string{
|
||||
aws.String(si.InstanceID),
|
||||
instTermWaiter := ec2.NewInstanceTerminatedWaiter(a.ec2)
|
||||
err = instTermWaiter.Wait(
|
||||
context.Background(),
|
||||
&ec2.DescribeInstancesInput{
|
||||
InstanceIds: []string{si.InstanceID},
|
||||
},
|
||||
})
|
||||
time.Hour,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -456,9 +508,12 @@ func (a *AWS) deleteLTIfExists(si *SecureInstance) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
_, err := a.ec2.DeleteLaunchTemplate(&ec2.DeleteLaunchTemplateInput{
|
||||
LaunchTemplateId: aws.String(si.LTID),
|
||||
})
|
||||
_, err := a.ec2.DeleteLaunchTemplate(
|
||||
context.Background(),
|
||||
&ec2.DeleteLaunchTemplateInput{
|
||||
LaunchTemplateId: aws.String(si.LTID),
|
||||
},
|
||||
)
|
||||
if err == nil {
|
||||
si.LTID = ""
|
||||
}
|
||||
|
|
@ -470,9 +525,12 @@ func (a *AWS) deleteSGIfExists(si *SecureInstance) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
_, err := a.ec2.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{
|
||||
GroupId: aws.String(si.SGID),
|
||||
})
|
||||
_, err := a.ec2.DeleteSecurityGroup(
|
||||
context.Background(),
|
||||
&ec2.DeleteSecurityGroupInput{
|
||||
GroupId: aws.String(si.SGID),
|
||||
},
|
||||
)
|
||||
if err == nil {
|
||||
si.SGID = ""
|
||||
}
|
||||
|
|
@ -480,7 +538,7 @@ func (a *AWS) deleteSGIfExists(si *SecureInstance) error {
|
|||
}
|
||||
|
||||
func (a *AWS) createFleet(input *ec2.CreateFleetInput) (*ec2.CreateFleetOutput, error) {
|
||||
createFleetOutput, err := a.ec2.CreateFleet(input)
|
||||
createFleetOutput, err := a.ec2.CreateFleet(context.Background(), input)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to create spot fleet: %w", err)
|
||||
}
|
||||
|
|
@ -488,7 +546,7 @@ func (a *AWS) createFleet(input *ec2.CreateFleetInput) (*ec2.CreateFleetOutput,
|
|||
if len(createFleetOutput.Errors) > 0 && createFleetOutput.Errors[0].ErrorCode == aws.String("UnfillableCapacity") {
|
||||
logrus.Warn("Received UnfillableCapacity from CreateFleet, retrying CreateFleet with OnDemand instance")
|
||||
input.SpotOptions = nil
|
||||
createFleetOutput, err = a.ec2.CreateFleet(input)
|
||||
createFleetOutput, err = a.ec2.CreateFleet(context.Background(), input)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to create on-demand fleet: %w", err)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue