diff --git a/cmd/image-builder/describeimg_test.go b/cmd/image-builder/describeimg_test.go index 618ca7e..ddc7f4c 100644 --- a/cmd/image-builder/describeimg_test.go +++ b/cmd/image-builder/describeimg_test.go @@ -15,7 +15,7 @@ func TestDescribeImage(t *testing.T) { restore := main.MockNewRepoRegistry(testrepos.New) defer restore() - res, err := main.GetOneImage("", "centos-9", "tar", "x86_64") + res, err := main.GetOneImage("centos-9", "tar", "x86_64", nil) assert.NoError(t, err) var buf bytes.Buffer diff --git a/cmd/image-builder/export_test.go b/cmd/image-builder/export_test.go index 4371dcf..28df1aa 100644 --- a/cmd/image-builder/export_test.go +++ b/cmd/image-builder/export_test.go @@ -44,7 +44,7 @@ func MockOsStderr(new io.Writer) (restore func()) { func MockNewRepoRegistry(f func() (*reporegistry.RepoRegistry, error)) (restore func()) { saved := newRepoRegistry - newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) { + newRepoRegistry = func(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) { if dataDir != "" { panic(fmt.Sprintf("cannot use custom dataDir %v in mock", dataDir)) } diff --git a/cmd/image-builder/filters.go b/cmd/image-builder/filters.go index 24388e2..80d876c 100644 --- a/cmd/image-builder/filters.go +++ b/cmd/image-builder/filters.go @@ -10,9 +10,9 @@ import ( "github.com/osbuild/images/pkg/imagefilter" ) -func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) { +func newImageFilterDefault(dataDir string, extraRepos []string) (*imagefilter.ImageFilter, error) { fac := distrofactory.NewDefault() - repos, err := newRepoRegistry(dataDir) + repos, err := newRepoRegistry(dataDir, extraRepos) if err != nil { return nil, err } @@ -20,9 +20,18 @@ func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) { return imagefilter.New(fac, repos) } +type repoOptions struct { + DataDir string + ExtraRepos []string +} + // should this be moved to images:imagefilter? -func getOneImage(dataDir, distroName, imgTypeStr, archStr string) (*imagefilter.Result, error) { - imageFilter, err := newImageFilterDefault(dataDir) +func getOneImage(distroName, imgTypeStr, archStr string, repoOpts *repoOptions) (*imagefilter.Result, error) { + if repoOpts == nil { + repoOpts = &repoOptions{} + } + + imageFilter, err := newImageFilterDefault(repoOpts.DataDir, repoOpts.ExtraRepos) if err != nil { return nil, err } diff --git a/cmd/image-builder/filters_test.go b/cmd/image-builder/filters_test.go index 0d84801..5a86e47 100644 --- a/cmd/image-builder/filters_test.go +++ b/cmd/image-builder/filters_test.go @@ -14,7 +14,6 @@ func TestGetOneImageHappy(t *testing.T) { restore := main.MockNewRepoRegistry(testrepos.New) defer restore() - dataDir := "" for _, tc := range []struct { distro, imgType, arch string }{ @@ -23,7 +22,7 @@ func TestGetOneImageHappy(t *testing.T) { {"distro:centos-9", "type:qcow2", "x86_64"}, {"distro:centos-9", "type:qcow2", "arch:x86_64"}, } { - res, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch) + res, err := main.GetOneImage(tc.distro, tc.imgType, tc.arch, nil) assert.NoError(t, err) assert.Equal(t, "centos-9", res.Distro.Name()) assert.Equal(t, "qcow2", res.ImgType.Name()) @@ -35,7 +34,6 @@ func TestGetOneImageError(t *testing.T) { restore := main.MockNewRepoRegistry(testrepos.New) defer restore() - dataDir := "" for _, tc := range []struct { distro, imgType, arch string expectedErr string @@ -49,7 +47,7 @@ func TestGetOneImageError(t *testing.T) { `cannot use globs in "centos*" when getting a single image`, }, } { - _, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch) + _, err := main.GetOneImage(tc.distro, tc.imgType, tc.arch, nil) assert.EqualError(t, err, tc.expectedErr) } } diff --git a/cmd/image-builder/list.go b/cmd/image-builder/list.go index 9bf06d9..9ef362b 100644 --- a/cmd/image-builder/list.go +++ b/cmd/image-builder/list.go @@ -4,8 +4,8 @@ import ( "github.com/osbuild/images/pkg/imagefilter" ) -func listImages(dataDir, output string, filterExprs []string) error { - imageFilter, err := newImageFilterDefault(dataDir) +func listImages(dataDir string, extraRepos []string, output string, filterExprs []string) error { + imageFilter, err := newImageFilterDefault(dataDir, extraRepos) if err != nil { return err } diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index e67beb1..dc1335b 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -45,8 +45,12 @@ func cmdListImages(cmd *cobra.Command, args []string) error { if err != nil { return err } + extraRepos, err := cmd.Flags().GetStringArray("extra-repo") + if err != nil { + return err + } - return listImages(dataDir, output, filter) + return listImages(dataDir, extraRepos, output, filter) } func ostreeImageOptions(cmd *cobra.Command) (*ostree.ImageOptions, error) { @@ -79,6 +83,10 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st if err != nil { return nil, err } + extraRepos, err := cmd.Flags().GetStringArray("extra-repo") + if err != nil { + return nil, err + } archStr, err := cmd.Flags().GetString("arch") if err != nil { return nil, err @@ -129,7 +137,11 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st pbar.SetPulseMsgf("Manifest generation step") pbar.SetMessagef("Building manifest for %s-%s", distroStr, imgTypeStr) - img, err := getOneImage(dataDir, distroStr, imgTypeStr, archStr) + repoOpts := &repoOptions{ + DataDir: dataDir, + ExtraRepos: extraRepos, + } + img, err := getOneImage(distroStr, imgTypeStr, archStr, repoOpts) if err != nil { return nil, err } @@ -146,7 +158,7 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st RpmDownloader: rpmDownloader, WithSBOM: withSBOM, } - err = generateManifest(dataDir, img, w, opts) + err = generateManifest(dataDir, extraRepos, img, w, opts) return img, err } @@ -276,7 +288,7 @@ func cmdDescribeImg(cmd *cobra.Command, args []string) error { archStr = arch.Current().String() } imgTypeStr := args[0] - res, err := getOneImage(dataDir, distroStr, imgTypeStr, archStr) + res, err := getOneImage(distroStr, imgTypeStr, archStr, &repoOptions{DataDir: dataDir}) if err != nil { return err } @@ -304,6 +316,7 @@ operating systems like Fedora, CentOS and RHEL with easy customizations support. SilenceErrors: true, } rootCmd.PersistentFlags().String("datadir", "", `Override the default data directory for e.g. custom repositories/*.json data`) + rootCmd.PersistentFlags().StringArray("extra-repo", nil, `Add an extra repository during build (will *not* be gpg checked and not be part of the final image)`) rootCmd.PersistentFlags().String("output-dir", "", `Put output into the specified directory`) rootCmd.PersistentFlags().BoolP("verbose", "v", false, `Switch to verbose mode`) rootCmd.SetOut(osStdout) diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go index 3386bf8..95af02d 100644 --- a/cmd/image-builder/main_test.go +++ b/cmd/image-builder/main_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "os" + "os/exec" "path/filepath" "slices" "strings" @@ -15,6 +16,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" testrepos "github.com/osbuild/images/test/data/repositories" @@ -599,3 +601,48 @@ func TestProgressFromCmd(t *testing.T) { assert.Equal(t, tc.expectedProgress, fmt.Sprintf("%T", pbar)) } } + +func TestManifestExtraRepo(t *testing.T) { + if testing.Short() { + t.Skip("manifest generation takes a while") + } + if !hasDepsolveDnf() { + t.Skip("no osbuild-depsolve-dnf binary found") + } + if _, err := exec.LookPath("createrepo_c"); err != nil { + t.Skip("need createrepo_c to run this test") + } + + localRepoDir := filepath.Join(t.TempDir(), "repo") + err := os.MkdirAll(localRepoDir, 0755) + assert.NoError(t, err) + err = exec.Command("cp", "-a", "../../test/data/rpm/dummy-1.0.0-0.noarch.rpm", localRepoDir).Run() + assert.NoError(t, err) + err = exec.Command("createrepo_c", localRepoDir).Run() + assert.NoError(t, err) + + pkgHelloBlueprint := `{ + "packages": [ + {"name":"dummy"} + ] + }` + restore := main.MockOsArgs([]string{ + "manifest", + "qcow2", + "--distro=centos-9", + fmt.Sprintf("--extra-repo=file://%s", localRepoDir), + "--blueprint", makeTestBlueprint(t, pkgHelloBlueprint), + }) + defer restore() + + var fakeStdout bytes.Buffer + restore = main.MockOsStdout(&fakeStdout) + defer restore() + + err = main.Run() + require.NoError(t, err) + + // our local repo got added + assert.Contains(t, fakeStdout.String(), `"path":"dummy-1.0.0-0.noarch.rpm"`) + assert.Contains(t, fakeStdout.String(), fmt.Sprintf(`"url":"file://%s"`, localRepoDir)) +} diff --git a/cmd/image-builder/manifest.go b/cmd/image-builder/manifest.go index eed1cee..18bf5e6 100644 --- a/cmd/image-builder/manifest.go +++ b/cmd/image-builder/manifest.go @@ -39,8 +39,8 @@ func sbomWriter(outputDir, filename string, content io.Reader) error { return nil } -func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error { - repos, err := newRepoRegistry(dataDir) +func generateManifest(dataDir string, extraRepos []string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error { + repos, err := newRepoRegistry(dataDir, extraRepos) if err != nil { return err } diff --git a/cmd/image-builder/repos.go b/cmd/image-builder/repos.go index 4a2122c..48fb293 100644 --- a/cmd/image-builder/repos.go +++ b/cmd/image-builder/repos.go @@ -1,10 +1,13 @@ package main import ( + "fmt" "io/fs" + "net/url" "github.com/osbuild/images/data/repositories" "github.com/osbuild/images/pkg/reporegistry" + "github.com/osbuild/images/pkg/rpmmd" ) // defaultDataDirs contains the default search paths to look for @@ -16,7 +19,41 @@ var defaultDataDirs = []string{ "/usr/share/image-builder/repositories", } -var newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) { +type repoConfig struct { + DataDir string + ExtraRepos []string +} + +func parseExtraRepo(extraRepo string) ([]rpmmd.RepoConfig, error) { + // We want to eventually support more URIs repos here: + // - config:/path/to/repo.json + // - copr:@osbuild/osbuild (with full gpg retrival via the copr API) + // But for now just default to base-urls + + baseURL, err := url.Parse(extraRepo) + if err != nil { + return nil, fmt.Errorf("cannot parse extra repo %w", err) + } + if baseURL.Scheme == "" { + return nil, fmt.Errorf(`scheme missing in %q, please prefix with e.g. file:`, extraRepo) + } + + // TODO: to support gpg checking we will need to add signing keys. + // We will eventually add support for our own "repo.json" format + // which is rich enough to contain gpg keys (and more). + checkGPG := false + return []rpmmd.RepoConfig{ + { + Id: baseURL.String(), + Name: baseURL.String(), + BaseURLs: []string{baseURL.String()}, + CheckGPG: &checkGPG, + CheckRepoGPG: &checkGPG, + }, + }, nil +} + +var newRepoRegistry = func(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) { var dataDirs []string if dataDir != "" { dataDirs = []string{dataDir} @@ -24,5 +61,29 @@ var newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) { dataDirs = defaultDataDirs } - return reporegistry.New(dataDirs, []fs.FS{repos.FS}) + conf, err := reporegistry.LoadAllRepositories(dataDirs, []fs.FS{repos.FS}) + if err != nil { + return nil, err + } + + // XXX: this should probably go into manifestgen.Options as + // a new Options.ExtraRepoConf eventually (just like OverrideRepos) + for _, repo := range extraRepos { + // XXX: this loads the extra repo unconditionally to all + // distro/arch versions. we do not know in advance where + // it belongs to + extraRepo, err := parseExtraRepo(repo) + if err != nil { + return nil, err + } + for _, repoArchConfigs := range conf { + for arch := range repoArchConfigs { + archCfg := repoArchConfigs[arch] + archCfg = append(archCfg, extraRepo...) + repoArchConfigs[arch] = archCfg + } + } + } + + return reporegistry.NewFromDistrosRepoConfigs(conf), nil } diff --git a/cmd/image-builder/repos_test.go b/cmd/image-builder/repos_test.go new file mode 100644 index 0000000..6a7ab49 --- /dev/null +++ b/cmd/image-builder/repos_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/osbuild/images/pkg/rpmmd" +) + +func TestParseExtraRepoHappy(t *testing.T) { + checkGPG := false + + cfg, err := parseExtraRepo("file:///path/to/repo") + assert.NoError(t, err) + assert.Equal(t, cfg, []rpmmd.RepoConfig{ + { + Id: "file:///path/to/repo", + Name: "file:///path/to/repo", + BaseURLs: []string{"file:///path/to/repo"}, + CheckGPG: &checkGPG, + CheckRepoGPG: &checkGPG, + }, + }) +} + +func TestParseExtraRepoSad(t *testing.T) { + _, err := parseExtraRepo("/just/a/path") + assert.EqualError(t, err, `scheme missing in "/just/a/path", please prefix with e.g. file:`) +} diff --git a/test/data/rpm/build-dummy-rpm.sh b/test/data/rpm/build-dummy-rpm.sh new file mode 100755 index 0000000..d1db626 --- /dev/null +++ b/test/data/rpm/build-dummy-rpm.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +# inspired by from https://github.com/osbuild/osbuild-composer/blob/bdd2014c4467e9325b29624439c98584addc681f/test/cases/api.sh#L230-L262 +# thanks Thozza! + +# make a dummy rpm for our tests +DUMMYRPMDIR=$(mktemp -d) +DUMMYSPECFILE="$DUMMYRPMDIR/dummy.spec" +pushd "$DUMMYRPMDIR" + +cat < "$DUMMYSPECFILE" +#----------- spec file starts --------------- +Name: dummy +Version: 1.0.0 +Release: 0 +BuildArch: noarch +Vendor: dummy +Summary: Provides %{name} +License: BSD +Provides: dummy + +%description +%{summary} + +%files +EOF + +mkdir -p "DUMMYRPMDIR/rpmbuild" +rpmbuild --quiet --define "_topdir $DUMMYRPMDIR/rpmbuild" -bb "$DUMMYSPECFILE" +popd + +echo "Done building dummy rpm in $DUMMYRPMDIR" +cp "$DUMMYRPMDIR"/rpmbuild/RPMS/noarch/dummy-*.noarch.rpm . diff --git a/test/data/rpm/dummy-1.0.0-0.noarch.rpm b/test/data/rpm/dummy-1.0.0-0.noarch.rpm new file mode 100644 index 0000000..a743c4e Binary files /dev/null and b/test/data/rpm/dummy-1.0.0-0.noarch.rpm differ