From 7f5c869cd23661ea3defa434d4dd5fa408a66463 Mon Sep 17 00:00:00 2001 From: Tom Gundersen Date: Tue, 19 Nov 2019 16:08:49 +0100 Subject: [PATCH] upload/aws: add a sample AWS upload client This commandline tools uploads a file to S3, as a proof of concept. All options are mandatory. Credentials are only read from the commandline and not from the environment or configuration files. The next step is to add support for importing from S3 to EC2, currently the images we produce cannot be imported as-is, so this requires more research. To try this out: create an S3 bucket, get your credentials and call the tool, passing any value as `key`. Note that if the key already exists, it will be overwritten. Signed-off-by: Tom Gundersen --- cmd/osbuild-upload-aws/main.go | 49 +++++++++ go.mod | 1 + go.sum | 4 + golang-github-osbuild-composer.spec | 1 + internal/awsupload/awsupload.go | 160 ++++++++++++++++++++++++++++ internal/distro/fedora30/os.go | 4 +- 6 files changed, 217 insertions(+), 2 deletions(-) create mode 100644 cmd/osbuild-upload-aws/main.go create mode 100644 internal/awsupload/awsupload.go diff --git a/cmd/osbuild-upload-aws/main.go b/cmd/osbuild-upload-aws/main.go new file mode 100644 index 000000000..32f0ffd6b --- /dev/null +++ b/cmd/osbuild-upload-aws/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "flag" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/osbuild/osbuild-composer/internal/awsupload" +) + +func main() { + var accessKeyID string + var secretAccessKey string + var region string + var bucketName string + var keyName string + var filename string + var imageName string + flag.StringVar(&accessKeyID, "access-key-id", "", "access key ID") + flag.StringVar(&secretAccessKey, "secret-access-key", "", "secret access key") + flag.StringVar(®ion, "region", "", "target region") + flag.StringVar(&bucketName, "bucket", "", "target S3 bucket name") + flag.StringVar(&keyName, "key", "", "target S3 key name") + flag.StringVar(&filename, "image", "", "image file to upload") + flag.StringVar(&imageName, "name", "", "AMI name") + flag.Parse() + + a, err := awsupload.New(region, accessKeyID, secretAccessKey) + if err != nil { + println(err.Error()) + return + } + + uploadOutput, err := a.Upload(filename, bucketName, keyName) + if err != nil { + println(err.Error()) + return + } + + fmt.Printf("file uploaded to %s\n", aws.StringValue(&uploadOutput.Location)) + + ami, err := a.Register(imageName, bucketName, keyName) + if err != nil { + println(err.Error()) + return + } + + fmt.Printf("AMI registered: %s\n", aws.StringValue(ami)) +} diff --git a/go.mod b/go.mod index 4e5aecd82..57b522486 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/osbuild/osbuild-composer go 1.12 require ( + github.com/aws/aws-sdk-go v1.25.37 github.com/Azure/azure-storage-blob-go v0.8.0 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f github.com/gobwas/glob v0.2.3 diff --git a/go.sum b/go.sum index 7f8b2a91b..163bc1388 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/aws/aws-sdk-go v1.25.37 h1:gBtB/F3dophWpsUQKN/Kni+JzYEH2mGHF4hWNtfED1w= +github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= @@ -8,6 +10,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= diff --git a/golang-github-osbuild-composer.spec b/golang-github-osbuild-composer.spec index c659b895b..b6f14eef2 100644 --- a/golang-github-osbuild-composer.spec +++ b/golang-github-osbuild-composer.spec @@ -21,6 +21,7 @@ Source0: %{gosource} BuildRequires: systemd-rpm-macros BuildRequires: systemd +BuildRequires: golang(github.com/aws/aws-sdk-go) BuildRequires: golang-github-azure-storage-blob-devel BuildRequires: golang(github.com/coreos/go-systemd/activation) BuildRequires: golang(github.com/google/uuid) diff --git a/internal/awsupload/awsupload.go b/internal/awsupload/awsupload.go new file mode 100644 index 000000000..429ca886b --- /dev/null +++ b/internal/awsupload/awsupload.go @@ -0,0 +1,160 @@ +package awsupload + +import ( + "os" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +type AWS struct { + uploader *s3manager.Uploader + importer *ec2.EC2 +} + +func New(region, accessKeyID, accessKey string) (*AWS, error) { + // Session credentials + creds := credentials.NewStaticCredentials(accessKeyID, accessKey, "") + + // Create a Session with a custom region + sess, err := session.NewSession(&aws.Config{ + Credentials: creds, + Region: aws.String(region), + }) + if err != nil { + return nil, err + } + + return &AWS{ + uploader: s3manager.NewUploader(sess), + importer: ec2.New(sess), + }, nil +} + +func (a *AWS) Upload(filename, bucket, key string) (*s3manager.UploadOutput, error) { + file, err := os.Open(filename) + if err != nil { + return nil, err + } + + return a.uploader.Upload( + &s3manager.UploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: file, + }, + ) +} + +// WaitUntilImportSnapshotCompleted uses the Amazon EC2 API operation +// DescribeImportSnapshots to wait for a condition to be met before returning. +// If the condition is not met within the max attempt window, an error will +// be returned. +func WaitUntilImportSnapshotTaskCompleted(c *ec2.EC2, input *ec2.DescribeImportSnapshotTasksInput) error { + return WaitUntilImportSnapshotTaskCompletedWithContext(c, aws.BackgroundContext(), input) +} + +// WaitUntilImportSnapshotCompletedWithContext is an extended version of +// WaitUntilImportSnapshotCompleted. With the support for passing in a +// context and options to configure the Waiter and the underlying request +// options. +// +// The context must be non-nil and will be used for request cancellation. If +// the context is nil a panic will occur. In the future the SDK may create +// sub-contexts for http.Requests. See https://golang.org/pkg/context/ +// for more information on using Contexts. +func WaitUntilImportSnapshotTaskCompletedWithContext(c *ec2.EC2, ctx aws.Context, input *ec2.DescribeImportSnapshotTasksInput, opts ...request.WaiterOption) error { + w := request.Waiter{ + Name: "WaitUntilImportSnapshotTaskCompleted", + MaxAttempts: 40, + Delay: request.ConstantWaiterDelay(15 * time.Second), + Acceptors: []request.WaiterAcceptor{ + { + State: request.SuccessWaiterState, + Matcher: request.PathAllWaiterMatch, Argument: "ImportSnapshotTasks[].SnapshotTaskDetail.Status", + Expected: "completed", + }, + }, + Logger: c.Config.Logger, + NewRequest: func(opts []request.Option) (*request.Request, error) { + var inCpy *ec2.DescribeImportSnapshotTasksInput + if input != nil { + tmp := *input + inCpy = &tmp + } + req, _ := c.DescribeImportSnapshotTasksRequest(inCpy) + req.SetContext(ctx) + req.ApplyOptions(opts...) + return req, nil + }, + } + w.ApplyOptions(opts...) + + return w.WaitWithContext(ctx) +} + +func (a *AWS) Register(name, bucket, key string) (*string, error) { + importTaskOutput, err := a.importer.ImportSnapshot( + &ec2.ImportSnapshotInput{ + DiskContainer: &ec2.SnapshotDiskContainer{ + UserBucket: &ec2.UserBucket{ + S3Bucket: aws.String(bucket), + S3Key: aws.String(key), + }, + }, + }, + ) + if err != nil { + return nil, err + } + + err = WaitUntilImportSnapshotTaskCompleted( + a.importer, + &ec2.DescribeImportSnapshotTasksInput{ + ImportTaskIds: []*string{ + importTaskOutput.ImportTaskId, + }, + }, + ) + if err != nil { + return nil, err + } + + importOutput, err := a.importer.DescribeImportSnapshotTasks( + &ec2.DescribeImportSnapshotTasksInput{ + ImportTaskIds: []*string{ + importTaskOutput.ImportTaskId, + }, + }, + ) + + snapshotId := importOutput.ImportSnapshotTasks[0].SnapshotTaskDetail.SnapshotId + + registerOutput, err := a.importer.RegisterImage( + &ec2.RegisterImageInput{ + Architecture: aws.String("x86_64"), + VirtualizationType: aws.String("hvm"), + Name: aws.String(name), + RootDeviceName: aws.String("/dev/sda1"), + EnaSupport: aws.Bool(true), + BlockDeviceMappings: []*ec2.BlockDeviceMapping{ + { + DeviceName: aws.String("/dev/sda1"), + Ebs: &ec2.EbsBlockDevice{ + SnapshotId: snapshotId, + }, + }, + }, + }, + ) + if err != nil { + return nil, err + } + + return registerOutput.ImageId, nil +} diff --git a/internal/distro/fedora30/os.go b/internal/distro/fedora30/os.go index 0664057a5..949da07b2 100644 --- a/internal/distro/fedora30/os.go +++ b/internal/distro/fedora30/os.go @@ -21,8 +21,8 @@ type output interface { func init() { distro.Register("fedora-30", &Fedora30{ - outputs: map[string]output { - "ami": &amiOutput{}, + outputs: map[string]output{ + "ami": &amiOutput{}, "ext4-filesystem": &ext4Output{}, "live-iso": &liveIsoOutput{}, "partitioned-disk": &diskOutput{},