diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index 98485fd..b961b4e 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -11,6 +11,7 @@ import ( "github.com/osbuild/images/pkg/arch" "github.com/osbuild/images/pkg/imagefilter" + "github.com/osbuild/images/pkg/osbuild" "github.com/osbuild/images/pkg/ostree" "github.com/osbuild/image-builder-cli/internal/blueprintload" @@ -83,6 +84,14 @@ func cmdManifestWrapper(cmd *cobra.Command, args []string, w io.Writer, archChec if err != nil { return nil, err } + useLibrepo, err := cmd.Flags().GetBool("use-librepo") + if err != nil { + return nil, err + } + var rpmDownloader osbuild.RpmDownloader + if useLibrepo { + rpmDownloader = osbuild.RpmDownloaderLibrepo + } blueprintPath, err := cmd.Flags().GetString("blueprint") if err != nil { @@ -111,7 +120,7 @@ func cmdManifestWrapper(cmd *cobra.Command, args []string, w io.Writer, archChec } } - err = generateManifest(dataDir, blueprintPath, res, w, ostreeImgOpts) + err = generateManifest(dataDir, blueprintPath, res, w, ostreeImgOpts, rpmDownloader) return res, err } @@ -185,6 +194,7 @@ operating sytsems like centos and RHEL with easy customizations support.`, manifestCmd.Flags().String("ostree-ref", "", `OSTREE reference`) manifestCmd.Flags().String("ostree-parent", "", `OSTREE parent`) manifestCmd.Flags().String("ostree-url", "", `OSTREE url`) + manifestCmd.Flags().Bool("use-librepo", false, `(experimental) use librepo to download packages, needs new osbuild`) rootCmd.AddCommand(manifestCmd) buildCmd := &cobra.Command{ diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go index dd256f1..c33da9e 100644 --- a/cmd/image-builder/main_test.go +++ b/cmd/image-builder/main_test.go @@ -162,29 +162,36 @@ func TestManifestIntegrationSmoke(t *testing.T) { restore := main.MockNewRepoRegistry(testrepos.New) defer restore() - restore = main.MockOsArgs([]string{ - "manifest", - "qcow2", - "--arch=x86_64", - "--distro=centos-9", - fmt.Sprintf("--blueprint=%s", makeTestBlueprint(t, testBlueprint)), - }) - defer restore() + for _, useLibrepo := range []bool{false, true} { + t.Run(fmt.Sprintf("use-librepo: %v", useLibrepo), func(t *testing.T) { + restore = main.MockOsArgs([]string{ + "manifest", + "qcow2", + "--arch=x86_64", + "--distro=centos-9", + fmt.Sprintf("--blueprint=%s", makeTestBlueprint(t, testBlueprint)), + fmt.Sprintf("--use-librepo=%v", useLibrepo), + }) + defer restore() - var fakeStdout bytes.Buffer - restore = main.MockOsStdout(&fakeStdout) - defer restore() + var fakeStdout bytes.Buffer + restore = main.MockOsStdout(&fakeStdout) + defer restore() - err := main.Run() - assert.NoError(t, err) + err := main.Run() + assert.NoError(t, err) - pipelineNames, err := manifesttest.PipelineNamesFrom(fakeStdout.Bytes()) - assert.NoError(t, err) - assert.Contains(t, pipelineNames, "qcow2") + pipelineNames, err := manifesttest.PipelineNamesFrom(fakeStdout.Bytes()) + assert.NoError(t, err) + assert.Contains(t, pipelineNames, "qcow2") - // XXX: provide helpers in manifesttest to extract this in a nicer way - assert.Contains(t, fakeStdout.String(), `{"type":"org.osbuild.users","options":{"users":{"alice":{}}}}`) - assert.Contains(t, fakeStdout.String(), `"image":{"name":"registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal"`) + // XXX: provide helpers in manifesttest to extract this in a nicer way + assert.Contains(t, fakeStdout.String(), `{"type":"org.osbuild.users","options":{"users":{"alice":{}}}}`) + assert.Contains(t, fakeStdout.String(), `"image":{"name":"registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal"`) + + assert.Equal(t, strings.Contains(fakeStdout.String(), "org.osbuild.librepo"), useLibrepo) + }) + } } func TestManifestIntegrationCrossArch(t *testing.T) { diff --git a/cmd/image-builder/manifest.go b/cmd/image-builder/manifest.go index d5de84b..d1e4a0c 100644 --- a/cmd/image-builder/manifest.go +++ b/cmd/image-builder/manifest.go @@ -5,20 +5,22 @@ import ( "github.com/osbuild/images/pkg/distro" "github.com/osbuild/images/pkg/imagefilter" + "github.com/osbuild/images/pkg/osbuild" "github.com/osbuild/images/pkg/ostree" "github.com/osbuild/image-builder-cli/internal/blueprintload" "github.com/osbuild/image-builder-cli/internal/manifestgen" ) -func generateManifest(dataDir, blueprintPath string, res *imagefilter.Result, output io.Writer, ostreeOpts *ostree.ImageOptions) error { +func generateManifest(dataDir, blueprintPath string, res *imagefilter.Result, output io.Writer, ostreeOpts *ostree.ImageOptions, rpmDownloader osbuild.RpmDownloader) error { repos, err := newRepoRegistry(dataDir) if err != nil { return err } // XXX: add --rpmmd/cachedir option like bib mg, err := manifestgen.New(repos, &manifestgen.Options{ - Output: output, + Output: output, + RpmDownloader: rpmDownloader, }) if err != nil { return err diff --git a/internal/manifestgen/manifestgen.go b/internal/manifestgen/manifestgen.go index 0bd7dbf..1c6a096 100644 --- a/internal/manifestgen/manifestgen.go +++ b/internal/manifestgen/manifestgen.go @@ -10,6 +10,7 @@ import ( "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/distro" "github.com/osbuild/images/pkg/dnfjson" + "github.com/osbuild/images/pkg/osbuild" "github.com/osbuild/images/pkg/ostree" "github.com/osbuild/images/pkg/reporegistry" "github.com/osbuild/images/pkg/rpmmd" @@ -100,6 +101,8 @@ type Options struct { Depsolver DepsolveFunc ContainerResolver ContainerResolverFunc CommitResolver CommitResolverFunc + + RpmDownloader osbuild.RpmDownloader } // Generator can generate an osbuild manifest from a given repository @@ -113,6 +116,8 @@ type Generator struct { commitResolver CommitResolverFunc reporegistry *reporegistry.RepoRegistry + + rpmDownloader osbuild.RpmDownloader } // New will create a new manifest generator @@ -128,6 +133,7 @@ func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, er depsolver: opts.Depsolver, containerResolver: opts.ContainerResolver, commitResolver: opts.CommitResolver, + rpmDownloader: opts.RpmDownloader, } if mg.out == nil { mg.out = os.Stdout @@ -177,7 +183,7 @@ func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgTy if err != nil { return err } - mf, err := preManifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfig) + mf, err := preManifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfig, mg.rpmDownloader) if err != nil { return err } diff --git a/internal/manifestgen/manifestgen_test.go b/internal/manifestgen/manifestgen_test.go index 91128cb..434ac0b 100644 --- a/internal/manifestgen/manifestgen_test.go +++ b/internal/manifestgen/manifestgen_test.go @@ -4,16 +4,19 @@ import ( "bytes" "crypto/sha256" "fmt" + "strings" "testing" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/osbuild/images/pkg/blueprint" "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/distro" "github.com/osbuild/images/pkg/distrofactory" "github.com/osbuild/images/pkg/imagefilter" + "github.com/osbuild/images/pkg/osbuild" "github.com/osbuild/images/pkg/ostree" "github.com/osbuild/images/pkg/rpmmd" testrepos "github.com/osbuild/images/test/data/repositories" @@ -45,30 +48,43 @@ func TestManifestGeneratorDepsolve(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 1, len(res)) - var osbuildManifest bytes.Buffer - opts := &manifestgen.Options{ - Output: &osbuildManifest, - Depsolver: fakeDepsolve, - CommitResolver: panicCommitResolver, - ContainerResolver: panicContainerResolver, + for _, useLibrepo := range []bool{false, true} { + t.Run(fmt.Sprintf("useLibrepo: %v", useLibrepo), func(t *testing.T) { + var rpmDownloader osbuild.RpmDownloader + if useLibrepo { + rpmDownloader = osbuild.RpmDownloaderLibrepo + } + + var osbuildManifest bytes.Buffer + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: panicCommitResolver, + ContainerResolver: panicContainerResolver, + + RpmDownloader: rpmDownloader, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + var bp blueprint.Blueprint + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) + require.NoError(t, err) + + pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes()) + assert.NoError(t, err) + assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames) + + // we expect at least a "kernel" package in the manifest, + // sadly the test distro does not really generate much here so we + // need to use this as a canary that resolving happend + // XXX: add testhelper to manifesttest for this + expectedSha256 := sha256For("kernel") + assert.Contains(t, osbuildManifest.String(), expectedSha256) + + assert.Equal(t, strings.Contains(osbuildManifest.String(), "org.osbuild.librepo"), useLibrepo) + }) } - mg, err := manifestgen.New(repos, opts) - assert.NoError(t, err) - assert.NotNil(t, mg) - var bp blueprint.Blueprint - err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) - assert.NoError(t, err) - - pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes()) - assert.NoError(t, err) - assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames) - - // we expect at least a "kernel" package in the manifest, - // sadly the test distro does not really generate much here so we - // need to use this as a canary that resolving happend - // XXX: add testhelper to manifesttest for this - expectedSha256 := sha256For("kernel") - assert.Contains(t, osbuildManifest.String(), expectedSha256) } func TestManifestGeneratorWithOstreeCommit(t *testing.T) { @@ -121,16 +137,25 @@ func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d depsolvedSets := make(map[string][]rpmmd.PackageSpec) repoSets := make(map[string][]rpmmd.RepoConfig) for name, pkgSets := range packageSets { + repoId := fmt.Sprintf("repo_id_%s", name) var resolvedSet []rpmmd.PackageSpec for _, pkgSet := range pkgSets { for _, pkgName := range pkgSet.Include { resolvedSet = append(resolvedSet, rpmmd.PackageSpec{ Name: pkgName, Checksum: sha256For(pkgName), + Path: fmt.Sprintf("path/%s.rpm", pkgName), + RepoID: repoId, }) } } depsolvedSets[name] = resolvedSet + repoSets[name] = []rpmmd.RepoConfig{ + { + Id: repoId, + Metalink: "http://example.com/metalink", + }, + } } return depsolvedSets, repoSets, nil }