main: add experimental --use-librepo to support librepo downloads

This commit switches to the librepo enabled `images` library via:
```
go mod -replace github.com/osbuild/iamges=github.com/mvo5/images@librepo-sources-osbuild1974
```
which in turn needs osbuild PR#1974.

With that it then adds a new `--use-librepo` switch that will
enable librepo based downloading so that people can play with
the new backend.

cmd/image-builder/main.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/osbuild/images/pkg/arch"
 	"github.com/osbuild/images/pkg/imagefilter"
+	"github.com/osbuild/images/pkg/osbuild"
 	"github.com/osbuild/images/pkg/ostree"
 
 	"github.com/osbuild/image-builder-cli/internal/blueprintload"
@@ -83,6 +84,14 @@ func cmdManifestWrapper(cmd *cobra.Command, args []string, w io.Writer, archChec
 	if err != nil {
 		return nil, err
 	}
+	useLibrepo, err := cmd.Flags().GetBool("use-librepo")
+	if err != nil {
+		return nil, err
+	}
+	var rpmDownloader osbuild.RpmDownloader
+	if useLibrepo {
+		rpmDownloader = osbuild.RpmDownloaderLibrepo
+	}
 
 	blueprintPath, err := cmd.Flags().GetString("blueprint")
 	if err != nil {
@@ -111,7 +120,7 @@ func cmdManifestWrapper(cmd *cobra.Command, args []string, w io.Writer, archChec
 		}
 	}
 
-	err = generateManifest(dataDir, blueprintPath, res, w, ostreeImgOpts)
+	err = generateManifest(dataDir, blueprintPath, res, w, ostreeImgOpts, rpmDownloader)
 	return res, err
 }
 
@@ -185,6 +194,7 @@ operating sytsems like centos and RHEL with easy customizations support.`,
 	manifestCmd.Flags().String("ostree-ref", "", `OSTREE reference`)
 	manifestCmd.Flags().String("ostree-parent", "", `OSTREE parent`)
 	manifestCmd.Flags().String("ostree-url", "", `OSTREE url`)
+	manifestCmd.Flags().Bool("use-librepo", false, `(experimental) use librepo to download packages, needs new osbuild`)
 	rootCmd.AddCommand(manifestCmd)
 
 	buildCmd := &cobra.Command{

cmd/image-builder/main_test.go

@@ -162,29 +162,36 @@ func TestManifestIntegrationSmoke(t *testing.T) {
 	restore := main.MockNewRepoRegistry(testrepos.New)
 	defer restore()
 
-	restore = main.MockOsArgs([]string{
+	for _, useLibrepo := range []bool{false, true} {
-		"manifest",
+		t.Run(fmt.Sprintf("use-librepo: %v", useLibrepo), func(t *testing.T) {
-		"qcow2",
+			restore = main.MockOsArgs([]string{
-		"--arch=x86_64",
+				"manifest",
-		"--distro=centos-9",
+				"qcow2",
-		fmt.Sprintf("--blueprint=%s", makeTestBlueprint(t, testBlueprint)),
+				"--arch=x86_64",
-	})
+				"--distro=centos-9",
-	defer restore()
+				fmt.Sprintf("--blueprint=%s", makeTestBlueprint(t, testBlueprint)),
+				fmt.Sprintf("--use-librepo=%v", useLibrepo),
+			})
+			defer restore()
 
-	var fakeStdout bytes.Buffer
+			var fakeStdout bytes.Buffer
-	restore = main.MockOsStdout(&fakeStdout)
+			restore = main.MockOsStdout(&fakeStdout)
-	defer restore()
+			defer restore()
 
-	err := main.Run()
+			err := main.Run()
-	assert.NoError(t, err)
+			assert.NoError(t, err)
 
-	pipelineNames, err := manifesttest.PipelineNamesFrom(fakeStdout.Bytes())
+			pipelineNames, err := manifesttest.PipelineNamesFrom(fakeStdout.Bytes())
-	assert.NoError(t, err)
+			assert.NoError(t, err)
-	assert.Contains(t, pipelineNames, "qcow2")
+			assert.Contains(t, pipelineNames, "qcow2")
 
-	// XXX: provide helpers in manifesttest to extract this in a nicer way
+			// XXX: provide helpers in manifesttest to extract this in a nicer way
-	assert.Contains(t, fakeStdout.String(), `{"type":"org.osbuild.users","options":{"users":{"alice":{}}}}`)
+			assert.Contains(t, fakeStdout.String(), `{"type":"org.osbuild.users","options":{"users":{"alice":{}}}}`)
-	assert.Contains(t, fakeStdout.String(), `"image":{"name":"registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal"`)
+			assert.Contains(t, fakeStdout.String(), `"image":{"name":"registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal"`)
+
+			assert.Equal(t, strings.Contains(fakeStdout.String(), "org.osbuild.librepo"), useLibrepo)
+		})
+	}
 }
 
 func TestManifestIntegrationCrossArch(t *testing.T) {

cmd/image-builder/manifest.go

@@ -5,20 +5,22 @@ import (
 
 	"github.com/osbuild/images/pkg/distro"
 	"github.com/osbuild/images/pkg/imagefilter"
+	"github.com/osbuild/images/pkg/osbuild"
 	"github.com/osbuild/images/pkg/ostree"
 
 	"github.com/osbuild/image-builder-cli/internal/blueprintload"
 	"github.com/osbuild/image-builder-cli/internal/manifestgen"
 )
 
-func generateManifest(dataDir, blueprintPath string, res *imagefilter.Result, output io.Writer, ostreeOpts *ostree.ImageOptions) error {
+func generateManifest(dataDir, blueprintPath string, res *imagefilter.Result, output io.Writer, ostreeOpts *ostree.ImageOptions, rpmDownloader osbuild.RpmDownloader) error {
 	repos, err := newRepoRegistry(dataDir)
 	if err != nil {
 		return err
 	}
 	// XXX: add --rpmmd/cachedir option like bib
 	mg, err := manifestgen.New(repos, &manifestgen.Options{
-		Output: output,
+		Output:        output,
+		RpmDownloader: rpmDownloader,
 	})
 	if err != nil {
 		return err

internal/manifestgen/manifestgen.go

@@ -10,6 +10,7 @@ import (
 	"github.com/osbuild/images/pkg/container"
 	"github.com/osbuild/images/pkg/distro"
 	"github.com/osbuild/images/pkg/dnfjson"
+	"github.com/osbuild/images/pkg/osbuild"
 	"github.com/osbuild/images/pkg/ostree"
 	"github.com/osbuild/images/pkg/reporegistry"
 	"github.com/osbuild/images/pkg/rpmmd"
@@ -100,6 +101,8 @@ type Options struct {
 	Depsolver         DepsolveFunc
 	ContainerResolver ContainerResolverFunc
 	CommitResolver    CommitResolverFunc
+
+	RpmDownloader osbuild.RpmDownloader
 }
 
 // Generator can generate an osbuild manifest from a given repository
@@ -113,6 +116,8 @@ type Generator struct {
 	commitResolver    CommitResolverFunc
 
 	reporegistry *reporegistry.RepoRegistry
+
+	rpmDownloader osbuild.RpmDownloader
 }
 
 // New will create a new manifest generator
@@ -128,6 +133,7 @@ func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, er
 		depsolver:         opts.Depsolver,
 		containerResolver: opts.ContainerResolver,
 		commitResolver:    opts.CommitResolver,
+		rpmDownloader:     opts.RpmDownloader,
 	}
 	if mg.out == nil {
 		mg.out = os.Stdout
@@ -177,7 +183,7 @@ func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgTy
 	if err != nil {
 		return err
 	}
-	mf, err := preManifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfig)
+	mf, err := preManifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfig, mg.rpmDownloader)
 	if err != nil {
 		return err
 	}

internal/manifestgen/manifestgen_test.go

@@ -4,16 +4,19 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/osbuild/images/pkg/blueprint"
 	"github.com/osbuild/images/pkg/container"
 	"github.com/osbuild/images/pkg/distro"
 	"github.com/osbuild/images/pkg/distrofactory"
 	"github.com/osbuild/images/pkg/imagefilter"
+	"github.com/osbuild/images/pkg/osbuild"
 	"github.com/osbuild/images/pkg/ostree"
 	"github.com/osbuild/images/pkg/rpmmd"
 	testrepos "github.com/osbuild/images/test/data/repositories"
@@ -45,30 +48,43 @@ func TestManifestGeneratorDepsolve(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(res))
 
-	var osbuildManifest bytes.Buffer
+	for _, useLibrepo := range []bool{false, true} {
-	opts := &manifestgen.Options{
+		t.Run(fmt.Sprintf("useLibrepo: %v", useLibrepo), func(t *testing.T) {
-		Output:            &osbuildManifest,
+			var rpmDownloader osbuild.RpmDownloader
-		Depsolver:         fakeDepsolve,
+			if useLibrepo {
-		CommitResolver:    panicCommitResolver,
+				rpmDownloader = osbuild.RpmDownloaderLibrepo
-		ContainerResolver: panicContainerResolver,
+			}
+
+			var osbuildManifest bytes.Buffer
+			opts := &manifestgen.Options{
+				Output:            &osbuildManifest,
+				Depsolver:         fakeDepsolve,
+				CommitResolver:    panicCommitResolver,
+				ContainerResolver: panicContainerResolver,
+
+				RpmDownloader: rpmDownloader,
+			}
+			mg, err := manifestgen.New(repos, opts)
+			assert.NoError(t, err)
+			assert.NotNil(t, mg)
+			var bp blueprint.Blueprint
+			err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
+			require.NoError(t, err)
+
+			pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes())
+			assert.NoError(t, err)
+			assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames)
+
+			// we expect at least a "kernel" package in the manifest,
+			// sadly the test distro does not really generate much here so we
+			// need to use this as a canary that resolving happend
+			// XXX: add testhelper to manifesttest for this
+			expectedSha256 := sha256For("kernel")
+			assert.Contains(t, osbuildManifest.String(), expectedSha256)
+
+			assert.Equal(t, strings.Contains(osbuildManifest.String(), "org.osbuild.librepo"), useLibrepo)
+		})
 	}
-	mg, err := manifestgen.New(repos, opts)
-	assert.NoError(t, err)
-	assert.NotNil(t, mg)
-	var bp blueprint.Blueprint
-	err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
-	assert.NoError(t, err)
-
-	pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes())
-	assert.NoError(t, err)
-	assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames)
-
-	// we expect at least a "kernel" package in the manifest,
-	// sadly the test distro does not really generate much here so we
-	// need to use this as a canary that resolving happend
-	// XXX: add testhelper to manifesttest for this
-	expectedSha256 := sha256For("kernel")
-	assert.Contains(t, osbuildManifest.String(), expectedSha256)
 }
 
 func TestManifestGeneratorWithOstreeCommit(t *testing.T) {
@@ -121,16 +137,25 @@ func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d
 	depsolvedSets := make(map[string][]rpmmd.PackageSpec)
 	repoSets := make(map[string][]rpmmd.RepoConfig)
 	for name, pkgSets := range packageSets {
+		repoId := fmt.Sprintf("repo_id_%s", name)
 		var resolvedSet []rpmmd.PackageSpec
 		for _, pkgSet := range pkgSets {
 			for _, pkgName := range pkgSet.Include {
 				resolvedSet = append(resolvedSet, rpmmd.PackageSpec{
 					Name:     pkgName,
 					Checksum: sha256For(pkgName),
+					Path:     fmt.Sprintf("path/%s.rpm", pkgName),
+					RepoID:   repoId,
 				})
 			}
 		}
 		depsolvedSets[name] = resolvedSet
+		repoSets[name] = []rpmmd.RepoConfig{
+			{
+				Id:       repoId,
+				Metalink: "http://example.com/metalink",
+			},
+		}
 	}
 	return depsolvedSets, repoSets, nil
 }