diff --git a/cmd/image-builder/export_test.go b/cmd/image-builder/export_test.go index 4b4e8ea..4371dcf 100644 --- a/cmd/image-builder/export_test.go +++ b/cmd/image-builder/export_test.go @@ -5,6 +5,8 @@ import ( "io" "os" + "github.com/osbuild/images/pkg/cloud" + "github.com/osbuild/images/pkg/cloud/awscloud" "github.com/osbuild/images/pkg/reporegistry" ) @@ -60,3 +62,11 @@ func MockDistroGetHostDistroName(f func() (string, error)) (restore func()) { distroGetHostDistroName = saved } } + +func MockAwscloudNewUploader(f func(string, string, string, *awscloud.UploaderOptions) (cloud.Uploader, error)) (restore func()) { + saved := awscloudNewUploader + awscloudNewUploader = f + return func() { + awscloudNewUploader = saved + } +} diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 7ca053e..dbd4e80 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -6,13 +6,16 @@ import ( "io" "os" "os/signal" + "path/filepath" "syscall" + "github.com/cheggaaa/pb/v3" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/osbuild/bootc-image-builder/bib/pkg/progress" "github.com/osbuild/images/pkg/arch" + "github.com/osbuild/images/pkg/cloud/awscloud" "github.com/osbuild/images/pkg/imagefilter" "github.com/osbuild/images/pkg/osbuild" "github.com/osbuild/images/pkg/ostree" @@ -173,6 +176,66 @@ func progressFromCmd(cmd *cobra.Command) (progress.ProgressBar, error) { return progress.New(progressType) } +var awscloudNewUploader = awscloud.NewUploader + +func cmdUploadAWS(cmd *cobra.Command, args []string) error { + amiName, err := cmd.Flags().GetString("aws-ami-name") + if err != nil { + return err + } + bucketName, err := cmd.Flags().GetString("aws-bucket") + if err != nil { + return err + } + region, err := cmd.Flags().GetString("aws-region") + if err != nil { + return err + } + + rawDiskPath := args[0] + // XXX: can we actually inspect the image or leave some artifacts? + if filepath.Ext(rawDiskPath) != ".raw" { + return fmt.Errorf("expecting a raw disk ending with '.raw', got %q", filepath.Base(rawDiskPath)) + } + + uploader, err := awscloudNewUploader(region, bucketName, amiName, nil) + if err != nil { + return err + } + f, err := os.Open(rawDiskPath) + if err != nil { + return err + } + defer f.Close() + + // setup basic progress + st, err := f.Stat() + if err != nil { + return fmt.Errorf("cannot stat upload: %v", err) + } + pbar := pb.New64(st.Size()) + pbar.Set(pb.Bytes, true) + pbar.SetWriter(osStdout) + r := pbar.NewProxyReader(f) + pbar.Start() + defer pbar.Finish() + + return uploader.UploadAndRegister(r, osStderr) +} + +func cmdUpload(cmd *cobra.Command, args []string) error { + uploadTo, err := cmd.Flags().GetString("to") + if err != nil { + return err + } + switch uploadTo { + case "aws": + return cmdUploadAWS(cmd, args) + default: + return fmt.Errorf("unsupported cloud %q", uploadTo) + } +} + func cmdBuild(cmd *cobra.Command, args []string) error { cacheDir, err := cmd.Flags().GetString("cache") if err != nil { @@ -300,6 +363,19 @@ operating systems like Fedora, CentOS and RHEL with easy customizations support. manifestCmd.Flags().Bool("with-sbom", false, `export SPDX SBOM document`) rootCmd.AddCommand(manifestCmd) + uploadCmd := &cobra.Command{ + Use: "upload ", + Short: "Upload the given image from ", + RunE: cmdUpload, + SilenceUsage: true, + Args: cobra.ExactArgs(1), + } + uploadCmd.Flags().String("aws-ami-name", "", "name for the AMI in AWS (only for type=ami)") + uploadCmd.Flags().String("aws-bucket", "", "target S3 bucket name for intermediate storage when creating AMI (only for type=ami)") + uploadCmd.Flags().String("aws-region", "", "target region for AWS uploads (only for type=ami)") + rootCmd.AddCommand(uploadCmd) + uploadCmd.Flags().String("to", "", "upload to the given cloud") + buildCmd := &cobra.Command{ Use: "build ", Short: "Build the given image-type, e.g. qcow2 (tip: combine with --distro, --arch)", diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go index ce5fa65..c8c6c12 100644 --- a/cmd/image-builder/main_test.go +++ b/cmd/image-builder/main_test.go @@ -15,8 +15,10 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" - //"github.com/osbuild/bootc-image-builder/bib/pkg/progress" + "github.com/osbuild/images/pkg/cloud" + "github.com/osbuild/images/pkg/cloud/awscloud" testrepos "github.com/osbuild/images/test/data/repositories" main "github.com/osbuild/image-builder-cli/cmd/image-builder" @@ -600,3 +602,64 @@ func TestProgressFromCmd(t *testing.T) { assert.Equal(t, tc.expectedProgress, fmt.Sprintf("%T", pbar)) } } + +type fakeAwsUploader struct { + checkCalls int + + uploadAndRegisterRead bytes.Buffer + uploadAndRegisterCalls int +} + +var _ = cloud.Uploader(&fakeAwsUploader{}) + +func (fa *fakeAwsUploader) Check(status io.Writer) error { + fa.checkCalls++ + return nil +} + +func (fa *fakeAwsUploader) UploadAndRegister(r io.Reader, status io.Writer) error { + fa.uploadAndRegisterCalls++ + _, err := io.Copy(&fa.uploadAndRegisterRead, r) + return err +} + +func TestUploadWithAWSMock(t *testing.T) { + fakeDiskContent := "fake-raw-img" + + fakeImageFilePath := filepath.Join(t.TempDir(), "disk.raw") + err := os.WriteFile(fakeImageFilePath, []byte(fakeDiskContent), 0644) + assert.NoError(t, err) + + var regionName, bucketName, amiName string + var fa fakeAwsUploader + restore := main.MockAwscloudNewUploader(func(region string, bucket string, ami string, opts *awscloud.UploaderOptions) (cloud.Uploader, error) { + regionName = region + bucketName = bucket + amiName = ami + return &fa, nil + }) + defer restore() + + var fakeStdout bytes.Buffer + restore = main.MockOsStdout(&fakeStdout) + defer restore() + + restore = main.MockOsArgs([]string{ + "upload", + "--to=aws", + "--aws-region=aws-region-1", + "--aws-bucket=aws-bucket-2", + "--aws-ami-name=aws-ami-3", + fakeImageFilePath, + }) + err = main.Run() + require.NoError(t, err) + + assert.Equal(t, regionName, "aws-region-1") + assert.Equal(t, bucketName, "aws-bucket-2") + assert.Equal(t, amiName, "aws-ami-3") + + assert.Equal(t, fakeDiskContent, fa.uploadAndRegisterRead.String()) + // progress was rendered + assert.Contains(t, fakeStdout.String(), "--] 100.00%") +} diff --git a/go.mod b/go.mod index 8748d51..f6c0ccb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ go 1.22.6 require ( github.com/BurntSushi/toml v1.4.0 + github.com/cheggaaa/pb/v3 v3.1.6 github.com/gobwas/glob v0.2.3 github.com/osbuild/bootc-image-builder/bib v0.0.0-20250205182004-b35eaa8a3a91 github.com/osbuild/images v0.116.0 @@ -22,7 +23,7 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect - github.com/cheggaaa/pb/v3 v3.1.6 // indirect + github.com/aws/aws-sdk-go v1.55.6 // indirect github.com/containerd/cgroups/v3 v3.0.3 // indirect github.com/containerd/errdefs v0.3.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect @@ -68,6 +69,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.11 // indirect diff --git a/go.sum b/go.sum index 41367df..89ea6d1 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= +github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -163,6 +165,10 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -425,6 +431,9 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=