upload/aws: add a sample AWS upload client

This commandline tools uploads a file to S3, as a proof of concept.

All options are mandatory. Credentials are only read from the
commandline and not from the environment or configuration files.

The next step is to add support for importing from S3 to EC2,
currently the images we produce cannot be imported as-is, so this
requires more research.

To try this out: create an S3 bucket, get your credentials and
call the tool, passing any value as `key`. Note that if the key
already exists, it will be overwritten.

Signed-off-by: Tom Gundersen <teg@jklm.no>

cmd/osbuild-upload-aws/main.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/osbuild/osbuild-composer/internal/awsupload"
+)
+
+func main() {
+	var accessKeyID string
+	var secretAccessKey string
+	var region string
+	var bucketName string
+	var keyName string
+	var filename string
+	var imageName string
+	flag.StringVar(&accessKeyID, "access-key-id", "", "access key ID")
+	flag.StringVar(&secretAccessKey, "secret-access-key", "", "secret access key")
+	flag.StringVar(&region, "region", "", "target region")
+	flag.StringVar(&bucketName, "bucket", "", "target S3 bucket name")
+	flag.StringVar(&keyName, "key", "", "target S3 key name")
+	flag.StringVar(&filename, "image", "", "image file to upload")
+	flag.StringVar(&imageName, "name", "", "AMI name")
+	flag.Parse()
+
+	a, err := awsupload.New(region, accessKeyID, secretAccessKey)
+	if err != nil {
+		println(err.Error())
+		return
+	}
+
+	uploadOutput, err := a.Upload(filename, bucketName, keyName)
+	if err != nil {
+		println(err.Error())
+		return
+	}
+
+	fmt.Printf("file uploaded to %s\n", aws.StringValue(&uploadOutput.Location))
+
+	ami, err := a.Register(imageName, bucketName, keyName)
+	if err != nil {
+		println(err.Error())
+		return
+	}
+
+	fmt.Printf("AMI registered: %s\n", aws.StringValue(ami))
+}

go.mod

@@ -3,6 +3,7 @@ module github.com/osbuild/osbuild-composer
 go 1.12
 
 require (
+	github.com/aws/aws-sdk-go v1.25.37
 	github.com/Azure/azure-storage-blob-go v0.8.0
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
 	github.com/gobwas/glob v0.2.3

go.sum

@@ -1,3 +1,5 @@
+github.com/aws/aws-sdk-go v1.25.37 h1:gBtB/F3dophWpsUQKN/Kni+JzYEH2mGHF4hWNtfED1w=
+github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
 github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
@@ -8,6 +10,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=

golang-github-osbuild-composer.spec

@@ -21,6 +21,7 @@ Source0:        %{gosource}
 
 BuildRequires:  systemd-rpm-macros
 BuildRequires:  systemd
+BuildRequires:  golang(github.com/aws/aws-sdk-go)
 BuildRequires:  golang-github-azure-storage-blob-devel
 BuildRequires:  golang(github.com/coreos/go-systemd/activation)
 BuildRequires:  golang(github.com/google/uuid)

internal/awsupload/awsupload.go

@@ -0,0 +1,160 @@
+package awsupload
+
+import (
+	"os"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+)
+
+type AWS struct {
+	uploader *s3manager.Uploader
+	importer *ec2.EC2
+}
+
+func New(region, accessKeyID, accessKey string) (*AWS, error) {
+	// Session credentials
+	creds := credentials.NewStaticCredentials(accessKeyID, accessKey, "")
+
+	// Create a Session with a custom region
+	sess, err := session.NewSession(&aws.Config{
+		Credentials: creds,
+		Region:      aws.String(region),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &AWS{
+		uploader: s3manager.NewUploader(sess),
+		importer: ec2.New(sess),
+	}, nil
+}
+
+func (a *AWS) Upload(filename, bucket, key string) (*s3manager.UploadOutput, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	return a.uploader.Upload(
+		&s3manager.UploadInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(key),
+			Body:   file,
+		},
+	)
+}
+
+// WaitUntilImportSnapshotCompleted uses the Amazon EC2 API operation
+// DescribeImportSnapshots to wait for a condition to be met before returning.
+// If the condition is not met within the max attempt window, an error will
+// be returned.
+func WaitUntilImportSnapshotTaskCompleted(c *ec2.EC2, input *ec2.DescribeImportSnapshotTasksInput) error {
+	return WaitUntilImportSnapshotTaskCompletedWithContext(c, aws.BackgroundContext(), input)
+}
+
+// WaitUntilImportSnapshotCompletedWithContext is an extended version of
+// WaitUntilImportSnapshotCompleted. With the support for passing in a
+// context and options to configure the Waiter and the underlying request
+// options.
+//
+// The context must be non-nil and will be used for request cancellation. If
+// the context is nil a panic will occur. In the future the SDK may create
+// sub-contexts for http.Requests. See https://golang.org/pkg/context/
+// for more information on using Contexts.
+func WaitUntilImportSnapshotTaskCompletedWithContext(c *ec2.EC2, ctx aws.Context, input *ec2.DescribeImportSnapshotTasksInput, opts ...request.WaiterOption) error {
+	w := request.Waiter{
+		Name:        "WaitUntilImportSnapshotTaskCompleted",
+		MaxAttempts: 40,
+		Delay:       request.ConstantWaiterDelay(15 * time.Second),
+		Acceptors: []request.WaiterAcceptor{
+			{
+				State:   request.SuccessWaiterState,
+				Matcher: request.PathAllWaiterMatch, Argument: "ImportSnapshotTasks[].SnapshotTaskDetail.Status",
+				Expected: "completed",
+			},
+		},
+		Logger: c.Config.Logger,
+		NewRequest: func(opts []request.Option) (*request.Request, error) {
+			var inCpy *ec2.DescribeImportSnapshotTasksInput
+			if input != nil {
+				tmp := *input
+				inCpy = &tmp
+			}
+			req, _ := c.DescribeImportSnapshotTasksRequest(inCpy)
+			req.SetContext(ctx)
+			req.ApplyOptions(opts...)
+			return req, nil
+		},
+	}
+	w.ApplyOptions(opts...)
+
+	return w.WaitWithContext(ctx)
+}
+
+func (a *AWS) Register(name, bucket, key string) (*string, error) {
+	importTaskOutput, err := a.importer.ImportSnapshot(
+		&ec2.ImportSnapshotInput{
+			DiskContainer: &ec2.SnapshotDiskContainer{
+				UserBucket: &ec2.UserBucket{
+					S3Bucket: aws.String(bucket),
+					S3Key:    aws.String(key),
+				},
+			},
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	err = WaitUntilImportSnapshotTaskCompleted(
+		a.importer,
+		&ec2.DescribeImportSnapshotTasksInput{
+			ImportTaskIds: []*string{
+				importTaskOutput.ImportTaskId,
+			},
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	importOutput, err := a.importer.DescribeImportSnapshotTasks(
+		&ec2.DescribeImportSnapshotTasksInput{
+			ImportTaskIds: []*string{
+				importTaskOutput.ImportTaskId,
+			},
+		},
+	)
+
+	snapshotId := importOutput.ImportSnapshotTasks[0].SnapshotTaskDetail.SnapshotId
+
+	registerOutput, err := a.importer.RegisterImage(
+		&ec2.RegisterImageInput{
+			Architecture:       aws.String("x86_64"),
+			VirtualizationType: aws.String("hvm"),
+			Name:               aws.String(name),
+			RootDeviceName:     aws.String("/dev/sda1"),
+			EnaSupport:         aws.Bool(true),
+			BlockDeviceMappings: []*ec2.BlockDeviceMapping{
+				{
+					DeviceName: aws.String("/dev/sda1"),
+					Ebs: &ec2.EbsBlockDevice{
+						SnapshotId: snapshotId,
+					},
+				},
+			},
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	return registerOutput.ImageId, nil
+}

internal/distro/fedora30/os.go

@@ -21,8 +21,8 @@ type output interface {
 
 func init() {
 	distro.Register("fedora-30", &Fedora30{
-		outputs: map[string]output {
-			"ami": &amiOutput{},
+		outputs: map[string]output{
+			"ami":              &amiOutput{},
 			"ext4-filesystem":  &ext4Output{},
 			"live-iso":         &liveIsoOutput{},
 			"partitioned-disk": &diskOutput{},