main: add new --extra-repo flag

This commit adds a  new flag `--extra-repo` that can be used
to point to a repository url that is added to the base
repositories when depsolving. Note that *no* gpg checking
will be performed for such repos as there is no way to
add gpg-keys (yet) via this mechanism.

This means that with a repo created with e.g. `createrepo_c` like
```console
$ mkdir repo
$ (cd repo && dnf download hello)
$ createrepo_c ./repo
```
and a blueprint like:
```toml
[[packages]]
name = "hello"
```
a manifest is generated that gets hello from this local repo:
```console
$ image-builder  --extra-repo file:$(pwd)/repo manifest qcow2 --distro centos-9 --blueprint ./bp.toml |jq|grep hello
          "path": "hello-2.12.1-5.fc41.x86_64.rpm",
```
Note that this is part of the base repositories so anything with a
higher version number will get pulled from the extra-repo, even
system libraries or kernels. Note also that this repository does
not become part of the image so after the image build all rpms
from there are not updated (unless of course the normal repos
have higher versions of them).

Note as well that there is no safeguard right now against adding
extra repos for the wrong version of the distro, i.e. one could
add an extra repo build against/for fedora-42 on a fedora-40 image
which most likely will break with bad depsolve errors. But that
is okay, this option is meant for advanced users and testing.

cmd/image-builder/describeimg_test.go

@@ -15,7 +15,7 @@ func TestDescribeImage(t *testing.T) {
 	restore := main.MockNewRepoRegistry(testrepos.New)
 	defer restore()
 
-	res, err := main.GetOneImage("", "centos-9", "tar", "x86_64")
+	res, err := main.GetOneImage("centos-9", "tar", "x86_64", nil)
 	assert.NoError(t, err)
 
 	var buf bytes.Buffer

cmd/image-builder/export_test.go

@@ -44,7 +44,7 @@ func MockOsStderr(new io.Writer) (restore func()) {
 
 func MockNewRepoRegistry(f func() (*reporegistry.RepoRegistry, error)) (restore func()) {
 	saved := newRepoRegistry
-	newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) {
+	newRepoRegistry = func(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) {
 		if dataDir != "" {
 			panic(fmt.Sprintf("cannot use custom dataDir %v in mock", dataDir))
 		}

cmd/image-builder/filters.go

@@ -10,9 +10,9 @@ import (
 	"github.com/osbuild/images/pkg/imagefilter"
 )
 
-func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) {
+func newImageFilterDefault(dataDir string, extraRepos []string) (*imagefilter.ImageFilter, error) {
 	fac := distrofactory.NewDefault()
-	repos, err := newRepoRegistry(dataDir)
+	repos, err := newRepoRegistry(dataDir, extraRepos)
 	if err != nil {
 		return nil, err
 	}
@@ -20,9 +20,18 @@ func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) {
 	return imagefilter.New(fac, repos)
 }
 
+type repoOptions struct {
+	DataDir    string
+	ExtraRepos []string
+}
+
 // should this be moved to images:imagefilter?
-func getOneImage(dataDir, distroName, imgTypeStr, archStr string) (*imagefilter.Result, error) {
-	imageFilter, err := newImageFilterDefault(dataDir)
+func getOneImage(distroName, imgTypeStr, archStr string, repoOpts *repoOptions) (*imagefilter.Result, error) {
+	if repoOpts == nil {
+		repoOpts = &repoOptions{}
+	}
+
+	imageFilter, err := newImageFilterDefault(repoOpts.DataDir, repoOpts.ExtraRepos)
 	if err != nil {
 		return nil, err
 	}

cmd/image-builder/filters_test.go

@@ -14,7 +14,6 @@ func TestGetOneImageHappy(t *testing.T) {
 	restore := main.MockNewRepoRegistry(testrepos.New)
 	defer restore()
 
-	dataDir := ""
 	for _, tc := range []struct {
 		distro, imgType, arch string
 	}{
@@ -23,7 +22,7 @@ func TestGetOneImageHappy(t *testing.T) {
 		{"distro:centos-9", "type:qcow2", "x86_64"},
 		{"distro:centos-9", "type:qcow2", "arch:x86_64"},
 	} {
-		res, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch)
+		res, err := main.GetOneImage(tc.distro, tc.imgType, tc.arch, nil)
 		assert.NoError(t, err)
 		assert.Equal(t, "centos-9", res.Distro.Name())
 		assert.Equal(t, "qcow2", res.ImgType.Name())
@@ -35,7 +34,6 @@ func TestGetOneImageError(t *testing.T) {
 	restore := main.MockNewRepoRegistry(testrepos.New)
 	defer restore()
 
-	dataDir := ""
 	for _, tc := range []struct {
 		distro, imgType, arch string
 		expectedErr           string
@@ -49,7 +47,7 @@ func TestGetOneImageError(t *testing.T) {
 			`cannot use globs in "centos*" when getting a single image`,
 		},
 	} {
-		_, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch)
+		_, err := main.GetOneImage(tc.distro, tc.imgType, tc.arch, nil)
 		assert.EqualError(t, err, tc.expectedErr)
 	}
 }

cmd/image-builder/list.go

@@ -4,8 +4,8 @@ import (
 	"github.com/osbuild/images/pkg/imagefilter"
 )
 
-func listImages(dataDir, output string, filterExprs []string) error {
-	imageFilter, err := newImageFilterDefault(dataDir)
+func listImages(dataDir string, extraRepos []string, output string, filterExprs []string) error {
+	imageFilter, err := newImageFilterDefault(dataDir, extraRepos)
 	if err != nil {
 		return err
 	}

cmd/image-builder/main.go

@@ -45,8 +45,12 @@ func cmdListImages(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
+	extraRepos, err := cmd.Flags().GetStringArray("extra-repo")
+	if err != nil {
+		return err
+	}
 
-	return listImages(dataDir, output, filter)
+	return listImages(dataDir, extraRepos, output, filter)
 }
 
 func ostreeImageOptions(cmd *cobra.Command) (*ostree.ImageOptions, error) {
@@ -79,6 +83,10 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 	if err != nil {
 		return nil, err
 	}
+	extraRepos, err := cmd.Flags().GetStringArray("extra-repo")
+	if err != nil {
+		return nil, err
+	}
 	archStr, err := cmd.Flags().GetString("arch")
 	if err != nil {
 		return nil, err
@@ -129,7 +137,11 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 	pbar.SetPulseMsgf("Manifest generation step")
 	pbar.SetMessagef("Building manifest for %s-%s", distroStr, imgTypeStr)
 
-	img, err := getOneImage(dataDir, distroStr, imgTypeStr, archStr)
+	repoOpts := &repoOptions{
+		DataDir:    dataDir,
+		ExtraRepos: extraRepos,
+	}
+	img, err := getOneImage(distroStr, imgTypeStr, archStr, repoOpts)
 	if err != nil {
 		return nil, err
 	}
@@ -146,7 +158,7 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 		RpmDownloader: rpmDownloader,
 		WithSBOM:      withSBOM,
 	}
-	err = generateManifest(dataDir, img, w, opts)
+	err = generateManifest(dataDir, extraRepos, img, w, opts)
 	return img, err
 }
 
@@ -276,7 +288,7 @@ func cmdDescribeImg(cmd *cobra.Command, args []string) error {
 		archStr = arch.Current().String()
 	}
 	imgTypeStr := args[0]
-	res, err := getOneImage(dataDir, distroStr, imgTypeStr, archStr)
+	res, err := getOneImage(distroStr, imgTypeStr, archStr, &repoOptions{DataDir: dataDir})
 	if err != nil {
 		return err
 	}
@@ -304,6 +316,7 @@ operating systems like Fedora, CentOS and RHEL with easy customizations support.
 		SilenceErrors: true,
 	}
 	rootCmd.PersistentFlags().String("datadir", "", `Override the default data directory for e.g. custom repositories/*.json data`)
+	rootCmd.PersistentFlags().StringArray("extra-repo", nil, `Add an extra repository during build (will *not* be gpg checked and not be part of the final image)`)
 	rootCmd.PersistentFlags().String("output-dir", "", `Put output into the specified directory`)
 	rootCmd.PersistentFlags().BoolP("verbose", "v", false, `Switch to verbose mode`)
 	rootCmd.SetOut(osStdout)

cmd/image-builder/main_test.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"slices"
 	"strings"
@@ -15,6 +16,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	testrepos "github.com/osbuild/images/test/data/repositories"
 
@@ -599,3 +601,48 @@ func TestProgressFromCmd(t *testing.T) {
 		assert.Equal(t, tc.expectedProgress, fmt.Sprintf("%T", pbar))
 	}
 }
+
+func TestManifestExtraRepo(t *testing.T) {
+	if testing.Short() {
+		t.Skip("manifest generation takes a while")
+	}
+	if !hasDepsolveDnf() {
+		t.Skip("no osbuild-depsolve-dnf binary found")
+	}
+	if _, err := exec.LookPath("createrepo_c"); err != nil {
+		t.Skip("need createrepo_c to run this test")
+	}
+
+	localRepoDir := filepath.Join(t.TempDir(), "repo")
+	err := os.MkdirAll(localRepoDir, 0755)
+	assert.NoError(t, err)
+	err = exec.Command("cp", "-a", "../../test/data/rpm/dummy-1.0.0-0.noarch.rpm", localRepoDir).Run()
+	assert.NoError(t, err)
+	err = exec.Command("createrepo_c", localRepoDir).Run()
+	assert.NoError(t, err)
+
+	pkgHelloBlueprint := `{
+          "packages": [
+            {"name":"dummy"}
+          ]
+        }`
+	restore := main.MockOsArgs([]string{
+		"manifest",
+		"qcow2",
+		"--distro=centos-9",
+		fmt.Sprintf("--extra-repo=file://%s", localRepoDir),
+		"--blueprint", makeTestBlueprint(t, pkgHelloBlueprint),
+	})
+	defer restore()
+
+	var fakeStdout bytes.Buffer
+	restore = main.MockOsStdout(&fakeStdout)
+	defer restore()
+
+	err = main.Run()
+	require.NoError(t, err)
+
+	// our local repo got added
+	assert.Contains(t, fakeStdout.String(), `"path":"dummy-1.0.0-0.noarch.rpm"`)
+	assert.Contains(t, fakeStdout.String(), fmt.Sprintf(`"url":"file://%s"`, localRepoDir))
+}

cmd/image-builder/manifest.go

@@ -39,8 +39,8 @@ func sbomWriter(outputDir, filename string, content io.Reader) error {
 	return nil
 }
 
-func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error {
-	repos, err := newRepoRegistry(dataDir)
+func generateManifest(dataDir string, extraRepos []string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error {
+	repos, err := newRepoRegistry(dataDir, extraRepos)
 	if err != nil {
 		return err
 	}

cmd/image-builder/repos.go

@@ -1,10 +1,13 @@
 package main
 
 import (
+	"fmt"
 	"io/fs"
+	"net/url"
 
 	"github.com/osbuild/images/data/repositories"
 	"github.com/osbuild/images/pkg/reporegistry"
+	"github.com/osbuild/images/pkg/rpmmd"
 )
 
 // defaultDataDirs contains the default search paths to look for
@@ -16,7 +19,41 @@ var defaultDataDirs = []string{
 	"/usr/share/image-builder/repositories",
 }
 
-var newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) {
+type repoConfig struct {
+	DataDir    string
+	ExtraRepos []string
+}
+
+func parseExtraRepo(extraRepo string) ([]rpmmd.RepoConfig, error) {
+	// We want to eventually support more URIs repos here:
+	// - config:/path/to/repo.json
+	// - copr:@osbuild/osbuild (with full gpg retrival via the copr API)
+	// But for now just default to base-urls
+
+	baseURL, err := url.Parse(extraRepo)
+	if err != nil {
+		return nil, fmt.Errorf("cannot parse extra repo %w", err)
+	}
+	if baseURL.Scheme == "" {
+		return nil, fmt.Errorf(`scheme missing in %q, please prefix with e.g. file:`, extraRepo)
+	}
+
+	// TODO: to support gpg checking we will need to add signing keys.
+	// We will eventually add support for our own "repo.json" format
+	// which is rich enough to contain gpg keys (and more).
+	checkGPG := false
+	return []rpmmd.RepoConfig{
+		{
+			Id:           baseURL.String(),
+			Name:         baseURL.String(),
+			BaseURLs:     []string{baseURL.String()},
+			CheckGPG:     &checkGPG,
+			CheckRepoGPG: &checkGPG,
+		},
+	}, nil
+}
+
+var newRepoRegistry = func(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) {
 	var dataDirs []string
 	if dataDir != "" {
 		dataDirs = []string{dataDir}
@@ -24,5 +61,29 @@ var newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) {
 		dataDirs = defaultDataDirs
 	}
 
-	return reporegistry.New(dataDirs, []fs.FS{repos.FS})
+	conf, err := reporegistry.LoadAllRepositories(dataDirs, []fs.FS{repos.FS})
+	if err != nil {
+		return nil, err
+	}
+
+	// XXX: this should probably go into manifestgen.Options as
+	// a new Options.ExtraRepoConf eventually (just like OverrideRepos)
+	for _, repo := range extraRepos {
+		// XXX: this loads the extra repo unconditionally to all
+		// distro/arch versions. we do not know in advance where
+		// it belongs to
+		extraRepo, err := parseExtraRepo(repo)
+		if err != nil {
+			return nil, err
+		}
+		for _, repoArchConfigs := range conf {
+			for arch := range repoArchConfigs {
+				archCfg := repoArchConfigs[arch]
+				archCfg = append(archCfg, extraRepo...)
+				repoArchConfigs[arch] = archCfg
+			}
+		}
+	}
+
+	return reporegistry.NewFromDistrosRepoConfigs(conf), nil
 }

cmd/image-builder/repos_test.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/osbuild/images/pkg/rpmmd"
+)
+
+func TestParseExtraRepoHappy(t *testing.T) {
+	checkGPG := false
+
+	cfg, err := parseExtraRepo("file:///path/to/repo")
+	assert.NoError(t, err)
+	assert.Equal(t, cfg, []rpmmd.RepoConfig{
+		{
+			Id:           "file:///path/to/repo",
+			Name:         "file:///path/to/repo",
+			BaseURLs:     []string{"file:///path/to/repo"},
+			CheckGPG:     &checkGPG,
+			CheckRepoGPG: &checkGPG,
+		},
+	})
+}
+
+func TestParseExtraRepoSad(t *testing.T) {
+	_, err := parseExtraRepo("/just/a/path")
+	assert.EqualError(t, err, `scheme missing in "/just/a/path", please prefix with e.g. file:`)
+}

test/data/rpm/build-dummy-rpm.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+
+# inspired by from https://github.com/osbuild/osbuild-composer/blob/bdd2014c4467e9325b29624439c98584addc681f/test/cases/api.sh#L230-L262
+# thanks Thozza!
+
+# make a dummy rpm for our tests
+DUMMYRPMDIR=$(mktemp -d)
+DUMMYSPECFILE="$DUMMYRPMDIR/dummy.spec"
+pushd "$DUMMYRPMDIR"
+
+cat <<EOF > "$DUMMYSPECFILE"
+#----------- spec file starts ---------------
+Name:                   dummy
+Version:                1.0.0
+Release:                0
+BuildArch:              noarch
+Vendor:                 dummy
+Summary:                Provides %{name}
+License:                BSD
+Provides:               dummy
+
+%description
+%{summary}
+
+%files
+EOF
+
+mkdir -p "DUMMYRPMDIR/rpmbuild"
+rpmbuild --quiet --define "_topdir $DUMMYRPMDIR/rpmbuild" -bb "$DUMMYSPECFILE"
+popd
+
+echo "Done building dummy rpm in $DUMMYRPMDIR"
+cp "$DUMMYRPMDIR"/rpmbuild/RPMS/noarch/dummy-*.noarch.rpm .