main: add --force-repo flag

This commit adds an `--force-repo` flag that can be used
to replace all the base repositories with a base url to
a repository. This is useful for testing but also dangerous
as it will not do any checks and happily use a fedora-42 repository
for centos-8 depsolving.

This will make the use-case of the koji builder easier and is
also something that the `build` tool in `images` supports.

cmd/image-builder/filters.go

@@ -21,8 +21,14 @@ func newImageFilterDefault(dataDir string, extraRepos []string) (*imagefilter.Im
 }
 
 type repoOptions struct {
-	DataDir    string
+	// DataDir contains the base dir for the repo definition search path
+	DataDir string
+
+	// ExtraRepos contains extra baseURLs that get added to the depsolving
 	ExtraRepos []string
+
+	// ForceRepos contains baseURLs that replace *all* base repositories
+	ForceRepos []string
 }
 
 // should this be moved to images:imagefilter?

cmd/image-builder/main.go

@@ -87,6 +87,10 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 	if err != nil {
 		return nil, err
 	}
+	forceRepos, err := cmd.Flags().GetStringArray("force-repo")
+	if err != nil {
+		return nil, err
+	}
 	archStr, err := cmd.Flags().GetString("arch")
 	if err != nil {
 		return nil, err
@@ -140,6 +144,7 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 	repoOpts := &repoOptions{
 		DataDir:    dataDir,
 		ExtraRepos: extraRepos,
+		ForceRepos: forceRepos,
 	}
 	img, err := getOneImage(distroStr, imgTypeStr, archStr, repoOpts)
 	if err != nil {
@@ -157,6 +162,8 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st
 		Ostree:        ostreeImgOpts,
 		RpmDownloader: rpmDownloader,
 		WithSBOM:      withSBOM,
+
+		ForceRepos: forceRepos,
 	}
 	err = generateManifest(dataDir, extraRepos, img, w, opts)
 	return img, err
@@ -317,6 +324,7 @@ operating systems like Fedora, CentOS and RHEL with easy customizations support.
 	}
 	rootCmd.PersistentFlags().String("datadir", "", `Override the default data directory for e.g. custom repositories/*.json data`)
 	rootCmd.PersistentFlags().StringArray("extra-repo", nil, `Add an extra repository during build (will *not* be gpg checked and not be part of the final image)`)
+	rootCmd.PersistentFlags().StringArray("force-repo", nil, `Override the base repositories during build (these will not be part of the final image)`)
 	rootCmd.PersistentFlags().String("output-dir", "", `Put output into the specified directory`)
 	rootCmd.PersistentFlags().BoolP("verbose", "v", false, `Switch to verbose mode`)
 	rootCmd.SetOut(osStdout)

cmd/image-builder/main_test.go

@@ -646,3 +646,40 @@ func TestManifestExtraRepo(t *testing.T) {
 	assert.Contains(t, fakeStdout.String(), `"path":"dummy-1.0.0-0.noarch.rpm"`)
 	assert.Contains(t, fakeStdout.String(), fmt.Sprintf(`"url":"file://%s"`, localRepoDir))
 }
+
+func TestManifestOverrideRepo(t *testing.T) {
+	if testing.Short() {
+		t.Skip("manifest generation takes a while")
+	}
+	if !hasDepsolveDnf() {
+		t.Skip("no osbuild-depsolve-dnf binary found")
+	}
+
+	var fakeStderr bytes.Buffer
+	restore := main.MockOsStderr(&fakeStderr)
+	defer restore()
+
+	restore = main.MockOsArgs([]string{
+		"manifest",
+		"qcow2",
+		"--distro=centos-9",
+		"--arch=x86_64",
+		"--force-repo=http://xxx.abcdefgh-no-such-host.com/repo",
+	})
+	defer restore()
+
+	// XXX: dnfjson is very chatty and puts a bunch of output on stderr
+	// we should probably silence this in images as its the job of the
+	// error to catpure this. Use CaptureStdio here to ensure we don't
+	// get noisy and confusing errors when this test runs.
+	var err error
+	testutil.CaptureStdio(t, func() {
+		err = main.Run()
+	})
+	assert.ErrorContains(t, err, "forced repo#0 xxx.abcdefgh-no-such-host.com/repo: http://xxx.abcdefgh-no-such-host.com/repo]: Cannot download repomd.xml")
+	// XXX: we should probably look into "images" here, there is a bunch
+	// of redundancy in the full error message:
+	//
+	// `error depsolving: running osbuild-depsolve-dnf failed:
+	// DNF error occurred: RepoError: There was a problem reading a repository: Failed to download metadata for repo '9828718901ab404ac1b600157aec1a8b19f4b2139e7756f347fb0ecc06c92929' [forced repo#0 xxx.abcdefgh-no-such-host.com/repo: http://xxx.abcdefgh-no-such-host.com/repo]: Cannot download repomd.xml: Cannot download repodata/repomd.xml: All mirrors were tried`
+}

cmd/image-builder/manifest.go

@@ -21,6 +21,8 @@ type manifestOptions struct {
 	Ostree        *ostree.ImageOptions
 	RpmDownloader osbuild.RpmDownloader
 	WithSBOM      bool
+
+	ForceRepos []string
 }
 
 func sbomWriter(outputDir, filename string, content io.Reader) error {
@@ -58,6 +60,13 @@ func generateManifest(dataDir string, extraRepos []string, img *imagefilter.Resu
 			return sbomWriter(outputDir, filename, content)
 		}
 	}
+	if len(opts.ForceRepos) > 0 {
+		forcedRepos, err := parseRepoURLs(opts.ForceRepos, "forced")
+		if err != nil {
+			return err
+		}
+		manifestGenOpts.OverrideRepos = forcedRepos
+	}
 
 	mg, err := manifestgen.New(repos, manifestGenOpts)
 	if err != nil {

cmd/image-builder/repos.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 
 	"github.com/osbuild/images/data/repositories"
+	"github.com/osbuild/images/pkg/arch"
 	"github.com/osbuild/images/pkg/reporegistry"
 	"github.com/osbuild/images/pkg/rpmmd"
 )
@@ -24,33 +25,37 @@ type repoConfig struct {
 	ExtraRepos []string
 }
 
-func parseExtraRepo(extraRepo string) ([]rpmmd.RepoConfig, error) {
+func parseRepoURLs(repoURLs []string, what string) ([]rpmmd.RepoConfig, error) {
-	// We want to eventually support more URIs repos here:
+	var repoConf []rpmmd.RepoConfig
-	// - config:/path/to/repo.json
-	// - copr:@osbuild/osbuild (with full gpg retrival via the copr API)
-	// But for now just default to base-urls
 
-	baseURL, err := url.Parse(extraRepo)
+	for i, repoURL := range repoURLs {
-	if err != nil {
+		// We want to eventually support more URIs repos here:
-		return nil, fmt.Errorf("cannot parse extra repo %w", err)
+		// - config:/path/to/repo.json
-	}
+		// - copr:@osbuild/osbuild (with full gpg retrival via the copr API)
-	if baseURL.Scheme == "" {
+		// But for now just default to base-urls
-		return nil, fmt.Errorf(`scheme missing in %q, please prefix with e.g. file:`, extraRepo)
-	}
 
-	// TODO: to support gpg checking we will need to add signing keys.
+		baseURL, err := url.Parse(repoURL)
-	// We will eventually add support for our own "repo.json" format
+		if err != nil {
-	// which is rich enough to contain gpg keys (and more).
+			return nil, fmt.Errorf("cannot parse extra repo %w", err)
-	checkGPG := false
+		}
-	return []rpmmd.RepoConfig{
+		if baseURL.Scheme == "" {
-		{
+			return nil, fmt.Errorf(`scheme missing in %q, please prefix with e.g. file:// or https://`, repoURL)
-			Id:           baseURL.String(),
+		}
-			Name:         baseURL.String(),
+
+		// TODO: to support gpg checking we will need to add signing keys.
+		// We will eventually add support for our own "repo.json" format
+		// which is rich enough to contain gpg keys (and more).
+		checkGPG := false
+		repoConf = append(repoConf, rpmmd.RepoConfig{
+			Id:           fmt.Sprintf("%s-repo-%v", what, i),
+			Name:         fmt.Sprintf("%s repo#%v %s%s", what, i, baseURL.Host, baseURL.Path),
 			BaseURLs:     []string{baseURL.String()},
 			CheckGPG:     &checkGPG,
 			CheckRepoGPG: &checkGPG,
-		},
+		})
-	}, nil
+	}
+
+	return repoConf, nil
 }
 
 func newRepoRegistryImpl(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) {
@@ -68,21 +73,19 @@ func newRepoRegistryImpl(dataDir string, extraRepos []string) (*reporegistry.Rep
 
 	// XXX: this should probably go into manifestgen.Options as
 	// a new Options.ExtraRepoConf eventually (just like OverrideRepos)
-	for _, repo := range extraRepos {
+	repoConf, err := parseRepoURLs(extraRepos, "extra")
-		// XXX: this loads the extra repo unconditionally to all
+	if err != nil {
-		// distro/arch versions. we do not know in advance where
+		return nil, err
-		// it belongs to
+	}
-		extraRepo, err := parseExtraRepo(repo)
+	// Only add extra repos for the host architecture. We do not support
-		if err != nil {
+	// cross-building (yet) so this is fine. Once we support cross-building
-			return nil, err
+	// this needs to move (probably into manifestgen) because at this
-		}
+	// level we we do not know (yet) what manifest we will generate.
-		for _, repoArchConfigs := range conf {
+	myArch := arch.Current().String()
-			for arch := range repoArchConfigs {
+	for _, repoArchConfigs := range conf {
-				archCfg := repoArchConfigs[arch]
+		archCfg := repoArchConfigs[myArch]
-				archCfg = append(archCfg, extraRepo...)
+		archCfg = append(archCfg, repoConf...)
-				repoArchConfigs[arch] = archCfg
+		repoArchConfigs[myArch] = archCfg
-			}
-		}
 	}
 
 	return reporegistry.NewFromDistrosRepoConfigs(conf), nil

cmd/image-builder/repos_test.go

@@ -8,23 +8,36 @@ import (
 	"github.com/osbuild/images/pkg/rpmmd"
 )
 
-func TestParseExtraRepoHappy(t *testing.T) {
+func TestParseRepoURLsHappy(t *testing.T) {
 	checkGPG := false
 
-	cfg, err := parseExtraRepo("file:///path/to/repo")
+	cfg, err := parseRepoURLs([]string{
+		"file:///path/to/repo",
+		"https://example.com/repo",
+	}, "forced")
 	assert.NoError(t, err)
 	assert.Equal(t, cfg, []rpmmd.RepoConfig{
 		{
-			Id:           "file:///path/to/repo",
+			Id:           "forced-repo-0",
-			Name:         "file:///path/to/repo",
+			Name:         "forced repo#0 /path/to/repo",
 			BaseURLs:     []string{"file:///path/to/repo"},
 			CheckGPG:     &checkGPG,
 			CheckRepoGPG: &checkGPG,
 		},
+		{
+			Id:           "forced-repo-1",
+			Name:         "forced repo#1 example.com/repo",
+			BaseURLs:     []string{"https://example.com/repo"},
+			CheckGPG:     &checkGPG,
+			CheckRepoGPG: &checkGPG,
+		},
 	})
 }
 
 func TestParseExtraRepoSad(t *testing.T) {
-	_, err := parseExtraRepo("/just/a/path")
+	_, err := parseRepoURLs([]string{"/just/a/path"}, "forced")
-	assert.EqualError(t, err, `scheme missing in "/just/a/path", please prefix with e.g. file:`)
+	assert.EqualError(t, err, `scheme missing in "/just/a/path", please prefix with e.g. file:// or https://`)
+
+	_, err = parseRepoURLs([]string{"https://example.com", "/just/a/path"}, "forced")
+	assert.EqualError(t, err, `scheme missing in "/just/a/path", please prefix with e.g. file:// or https://`)
 }