From e41377b82a1a30659431d958e2c59797c9df74d8 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Wed, 29 Jan 2025 17:46:53 +0100 Subject: [PATCH] main: add new upload command This commit adds a new `upload` command that can be used to upload a raw image to the cloud. Currently only AWS is supported but as images adds more clouds to the uploader interfac we can easily expand more. The cloud is currently detected via the file extension, that mapping probably should also go into the `images` library. --- cmd/image-builder/export_test.go | 10 +++++ cmd/image-builder/main.go | 76 ++++++++++++++++++++++++++++++++ cmd/image-builder/main_test.go | 65 ++++++++++++++++++++++++++- go.mod | 4 +- go.sum | 9 ++++ 5 files changed, 162 insertions(+), 2 deletions(-) diff --git a/cmd/image-builder/export_test.go b/cmd/image-builder/export_test.go index 4b4e8ea..4371dcf 100644 --- a/cmd/image-builder/export_test.go +++ b/cmd/image-builder/export_test.go @@ -5,6 +5,8 @@ import ( "io" "os" + "github.com/osbuild/images/pkg/cloud" + "github.com/osbuild/images/pkg/cloud/awscloud" "github.com/osbuild/images/pkg/reporegistry" ) @@ -60,3 +62,11 @@ func MockDistroGetHostDistroName(f func() (string, error)) (restore func()) { distroGetHostDistroName = saved } } + +func MockAwscloudNewUploader(f func(string, string, string, *awscloud.UploaderOptions) (cloud.Uploader, error)) (restore func()) { + saved := awscloudNewUploader + awscloudNewUploader = f + return func() { + awscloudNewUploader = saved + } +} diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 7ca053e..dbd4e80 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -6,13 +6,16 @@ import ( "io" "os" "os/signal" + "path/filepath" "syscall" + "github.com/cheggaaa/pb/v3" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/osbuild/bootc-image-builder/bib/pkg/progress" "github.com/osbuild/images/pkg/arch" + "github.com/osbuild/images/pkg/cloud/awscloud" "github.com/osbuild/images/pkg/imagefilter" "github.com/osbuild/images/pkg/osbuild" "github.com/osbuild/images/pkg/ostree" @@ -173,6 +176,66 @@ func progressFromCmd(cmd *cobra.Command) (progress.ProgressBar, error) { return progress.New(progressType) } +var awscloudNewUploader = awscloud.NewUploader + +func cmdUploadAWS(cmd *cobra.Command, args []string) error { + amiName, err := cmd.Flags().GetString("aws-ami-name") + if err != nil { + return err + } + bucketName, err := cmd.Flags().GetString("aws-bucket") + if err != nil { + return err + } + region, err := cmd.Flags().GetString("aws-region") + if err != nil { + return err + } + + rawDiskPath := args[0] + // XXX: can we actually inspect the image or leave some artifacts? + if filepath.Ext(rawDiskPath) != ".raw" { + return fmt.Errorf("expecting a raw disk ending with '.raw', got %q", filepath.Base(rawDiskPath)) + } + + uploader, err := awscloudNewUploader(region, bucketName, amiName, nil) + if err != nil { + return err + } + f, err := os.Open(rawDiskPath) + if err != nil { + return err + } + defer f.Close() + + // setup basic progress + st, err := f.Stat() + if err != nil { + return fmt.Errorf("cannot stat upload: %v", err) + } + pbar := pb.New64(st.Size()) + pbar.Set(pb.Bytes, true) + pbar.SetWriter(osStdout) + r := pbar.NewProxyReader(f) + pbar.Start() + defer pbar.Finish() + + return uploader.UploadAndRegister(r, osStderr) +} + +func cmdUpload(cmd *cobra.Command, args []string) error { + uploadTo, err := cmd.Flags().GetString("to") + if err != nil { + return err + } + switch uploadTo { + case "aws": + return cmdUploadAWS(cmd, args) + default: + return fmt.Errorf("unsupported cloud %q", uploadTo) + } +} + func cmdBuild(cmd *cobra.Command, args []string) error { cacheDir, err := cmd.Flags().GetString("cache") if err != nil { @@ -300,6 +363,19 @@ operating systems like Fedora, CentOS and RHEL with easy customizations support. manifestCmd.Flags().Bool("with-sbom", false, `export SPDX SBOM document`) rootCmd.AddCommand(manifestCmd) + uploadCmd := &cobra.Command{ + Use: "upload ", + Short: "Upload the given image from ", + RunE: cmdUpload, + SilenceUsage: true, + Args: cobra.ExactArgs(1), + } + uploadCmd.Flags().String("aws-ami-name", "", "name for the AMI in AWS (only for type=ami)") + uploadCmd.Flags().String("aws-bucket", "", "target S3 bucket name for intermediate storage when creating AMI (only for type=ami)") + uploadCmd.Flags().String("aws-region", "", "target region for AWS uploads (only for type=ami)") + rootCmd.AddCommand(uploadCmd) + uploadCmd.Flags().String("to", "", "upload to the given cloud") + buildCmd := &cobra.Command{ Use: "build ", Short: "Build the given image-type, e.g. qcow2 (tip: combine with --distro, --arch)", diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go index ce5fa65..c8c6c12 100644 --- a/cmd/image-builder/main_test.go +++ b/cmd/image-builder/main_test.go @@ -15,8 +15,10 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" - //"github.com/osbuild/bootc-image-builder/bib/pkg/progress" + "github.com/osbuild/images/pkg/cloud" + "github.com/osbuild/images/pkg/cloud/awscloud" testrepos "github.com/osbuild/images/test/data/repositories" main "github.com/osbuild/image-builder-cli/cmd/image-builder" @@ -600,3 +602,64 @@ func TestProgressFromCmd(t *testing.T) { assert.Equal(t, tc.expectedProgress, fmt.Sprintf("%T", pbar)) } } + +type fakeAwsUploader struct { + checkCalls int + + uploadAndRegisterRead bytes.Buffer + uploadAndRegisterCalls int +} + +var _ = cloud.Uploader(&fakeAwsUploader{}) + +func (fa *fakeAwsUploader) Check(status io.Writer) error { + fa.checkCalls++ + return nil +} + +func (fa *fakeAwsUploader) UploadAndRegister(r io.Reader, status io.Writer) error { + fa.uploadAndRegisterCalls++ + _, err := io.Copy(&fa.uploadAndRegisterRead, r) + return err +} + +func TestUploadWithAWSMock(t *testing.T) { + fakeDiskContent := "fake-raw-img" + + fakeImageFilePath := filepath.Join(t.TempDir(), "disk.raw") + err := os.WriteFile(fakeImageFilePath, []byte(fakeDiskContent), 0644) + assert.NoError(t, err) + + var regionName, bucketName, amiName string + var fa fakeAwsUploader + restore := main.MockAwscloudNewUploader(func(region string, bucket string, ami string, opts *awscloud.UploaderOptions) (cloud.Uploader, error) { + regionName = region + bucketName = bucket + amiName = ami + return &fa, nil + }) + defer restore() + + var fakeStdout bytes.Buffer + restore = main.MockOsStdout(&fakeStdout) + defer restore() + + restore = main.MockOsArgs([]string{ + "upload", + "--to=aws", + "--aws-region=aws-region-1", + "--aws-bucket=aws-bucket-2", + "--aws-ami-name=aws-ami-3", + fakeImageFilePath, + }) + err = main.Run() + require.NoError(t, err) + + assert.Equal(t, regionName, "aws-region-1") + assert.Equal(t, bucketName, "aws-bucket-2") + assert.Equal(t, amiName, "aws-ami-3") + + assert.Equal(t, fakeDiskContent, fa.uploadAndRegisterRead.String()) + // progress was rendered + assert.Contains(t, fakeStdout.String(), "--] 100.00%") +} diff --git a/go.mod b/go.mod index 8748d51..f6c0ccb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ go 1.22.6 require ( github.com/BurntSushi/toml v1.4.0 + github.com/cheggaaa/pb/v3 v3.1.6 github.com/gobwas/glob v0.2.3 github.com/osbuild/bootc-image-builder/bib v0.0.0-20250205182004-b35eaa8a3a91 github.com/osbuild/images v0.116.0 @@ -22,7 +23,7 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/cheggaaa/pb/v3 v3.1.6 // indirect + github.com/aws/aws-sdk-go v1.55.6 // indirect github.com/containerd/cgroups/v3 v3.0.3 // indirect github.com/containerd/errdefs v0.3.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect @@ -68,6 +69,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.11 // indirect diff --git a/go.sum b/go.sum index 41367df..89ea6d1 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= +github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -163,6 +165,10 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -425,6 +431,9 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=