From a11e124133edfda5395d82627ca67ef939f9fbce Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Fri, 31 Jan 2025 12:56:53 +0100 Subject: [PATCH] main: add new `--extra-repo` flag This commit adds a new flag `--extra-repo` that can be used to point to a repository url that is added to the base repositories when depsolving. Note that *no* gpg checking will be performed for such repos as there is no way to add gpg-keys (yet) via this mechanism. This means that with a repo created with e.g. `createrepo_c` like ```console $ mkdir repo $ (cd repo && dnf download hello) $ createrepo_c ./repo ``` and a blueprint like: ```toml [[packages]] name = "hello" ``` a manifest is generated that gets hello from this local repo: ```console $ image-builder --extra-repo file:$(pwd)/repo manifest qcow2 --distro centos-9 --blueprint ./bp.toml |jq|grep hello "path": "hello-2.12.1-5.fc41.x86_64.rpm", ``` Note that this is part of the base repositories so anything with a higher version number will get pulled from the extra-repo, even system libraries or kernels. Note also that this repository does not become part of the image so after the image build all rpms from there are not updated (unless of course the normal repos have higher versions of them). Note as well that there is no safeguard right now against adding extra repos for the wrong version of the distro, i.e. one could add an extra repo build against/for fedora-42 on a fedora-40 image which most likely will break with bad depsolve errors. But that is okay, this option is meant for advanced users and testing. --- cmd/image-builder/describeimg_test.go | 2 +- cmd/image-builder/export_test.go | 2 +- cmd/image-builder/filters.go | 17 +++++-- cmd/image-builder/filters_test.go | 6 +-- cmd/image-builder/list.go | 4 +- cmd/image-builder/main.go | 21 ++++++-- cmd/image-builder/main_test.go | 47 ++++++++++++++++++ cmd/image-builder/manifest.go | 4 +- cmd/image-builder/repos.go | 65 ++++++++++++++++++++++++- cmd/image-builder/repos_test.go | 30 ++++++++++++ test/data/rpm/build-dummy-rpm.sh | 35 +++++++++++++ test/data/rpm/dummy-1.0.0-0.noarch.rpm | Bin 0 -> 5565 bytes 12 files changed, 213 insertions(+), 20 deletions(-) create mode 100644 cmd/image-builder/repos_test.go create mode 100755 test/data/rpm/build-dummy-rpm.sh create mode 100644 test/data/rpm/dummy-1.0.0-0.noarch.rpm diff --git a/cmd/image-builder/describeimg_test.go b/cmd/image-builder/describeimg_test.go index 618ca7e..ddc7f4c 100644 --- a/cmd/image-builder/describeimg_test.go +++ b/cmd/image-builder/describeimg_test.go @@ -15,7 +15,7 @@ func TestDescribeImage(t *testing.T) { restore := main.MockNewRepoRegistry(testrepos.New) defer restore() - res, err := main.GetOneImage("", "centos-9", "tar", "x86_64") + res, err := main.GetOneImage("centos-9", "tar", "x86_64", nil) assert.NoError(t, err) var buf bytes.Buffer diff --git a/cmd/image-builder/export_test.go b/cmd/image-builder/export_test.go index 4371dcf..28df1aa 100644 --- a/cmd/image-builder/export_test.go +++ b/cmd/image-builder/export_test.go @@ -44,7 +44,7 @@ func MockOsStderr(new io.Writer) (restore func()) { func MockNewRepoRegistry(f func() (*reporegistry.RepoRegistry, error)) (restore func()) { saved := newRepoRegistry - newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) { + newRepoRegistry = func(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) { if dataDir != "" { panic(fmt.Sprintf("cannot use custom dataDir %v in mock", dataDir)) } diff --git a/cmd/image-builder/filters.go b/cmd/image-builder/filters.go index 24388e2..80d876c 100644 --- a/cmd/image-builder/filters.go +++ b/cmd/image-builder/filters.go @@ -10,9 +10,9 @@ import ( "github.com/osbuild/images/pkg/imagefilter" ) -func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) { +func newImageFilterDefault(dataDir string, extraRepos []string) (*imagefilter.ImageFilter, error) { fac := distrofactory.NewDefault() - repos, err := newRepoRegistry(dataDir) + repos, err := newRepoRegistry(dataDir, extraRepos) if err != nil { return nil, err } @@ -20,9 +20,18 @@ func newImageFilterDefault(dataDir string) (*imagefilter.ImageFilter, error) { return imagefilter.New(fac, repos) } +type repoOptions struct { + DataDir string + ExtraRepos []string +} + // should this be moved to images:imagefilter? -func getOneImage(dataDir, distroName, imgTypeStr, archStr string) (*imagefilter.Result, error) { - imageFilter, err := newImageFilterDefault(dataDir) +func getOneImage(distroName, imgTypeStr, archStr string, repoOpts *repoOptions) (*imagefilter.Result, error) { + if repoOpts == nil { + repoOpts = &repoOptions{} + } + + imageFilter, err := newImageFilterDefault(repoOpts.DataDir, repoOpts.ExtraRepos) if err != nil { return nil, err } diff --git a/cmd/image-builder/filters_test.go b/cmd/image-builder/filters_test.go index 0d84801..5a86e47 100644 --- a/cmd/image-builder/filters_test.go +++ b/cmd/image-builder/filters_test.go @@ -14,7 +14,6 @@ func TestGetOneImageHappy(t *testing.T) { restore := main.MockNewRepoRegistry(testrepos.New) defer restore() - dataDir := "" for _, tc := range []struct { distro, imgType, arch string }{ @@ -23,7 +22,7 @@ func TestGetOneImageHappy(t *testing.T) { {"distro:centos-9", "type:qcow2", "x86_64"}, {"distro:centos-9", "type:qcow2", "arch:x86_64"}, } { - res, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch) + res, err := main.GetOneImage(tc.distro, tc.imgType, tc.arch, nil) assert.NoError(t, err) assert.Equal(t, "centos-9", res.Distro.Name()) assert.Equal(t, "qcow2", res.ImgType.Name()) @@ -35,7 +34,6 @@ func TestGetOneImageError(t *testing.T) { restore := main.MockNewRepoRegistry(testrepos.New) defer restore() - dataDir := "" for _, tc := range []struct { distro, imgType, arch string expectedErr string @@ -49,7 +47,7 @@ func TestGetOneImageError(t *testing.T) { `cannot use globs in "centos*" when getting a single image`, }, } { - _, err := main.GetOneImage(dataDir, tc.distro, tc.imgType, tc.arch) + _, err := main.GetOneImage(tc.distro, tc.imgType, tc.arch, nil) assert.EqualError(t, err, tc.expectedErr) } } diff --git a/cmd/image-builder/list.go b/cmd/image-builder/list.go index 9bf06d9..9ef362b 100644 --- a/cmd/image-builder/list.go +++ b/cmd/image-builder/list.go @@ -4,8 +4,8 @@ import ( "github.com/osbuild/images/pkg/imagefilter" ) -func listImages(dataDir, output string, filterExprs []string) error { - imageFilter, err := newImageFilterDefault(dataDir) +func listImages(dataDir string, extraRepos []string, output string, filterExprs []string) error { + imageFilter, err := newImageFilterDefault(dataDir, extraRepos) if err != nil { return err } diff --git a/cmd/image-builder/main.go b/cmd/image-builder/main.go index e67beb1..dc1335b 100644 --- a/cmd/image-builder/main.go +++ b/cmd/image-builder/main.go @@ -45,8 +45,12 @@ func cmdListImages(cmd *cobra.Command, args []string) error { if err != nil { return err } + extraRepos, err := cmd.Flags().GetStringArray("extra-repo") + if err != nil { + return err + } - return listImages(dataDir, output, filter) + return listImages(dataDir, extraRepos, output, filter) } func ostreeImageOptions(cmd *cobra.Command) (*ostree.ImageOptions, error) { @@ -79,6 +83,10 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st if err != nil { return nil, err } + extraRepos, err := cmd.Flags().GetStringArray("extra-repo") + if err != nil { + return nil, err + } archStr, err := cmd.Flags().GetString("arch") if err != nil { return nil, err @@ -129,7 +137,11 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st pbar.SetPulseMsgf("Manifest generation step") pbar.SetMessagef("Building manifest for %s-%s", distroStr, imgTypeStr) - img, err := getOneImage(dataDir, distroStr, imgTypeStr, archStr) + repoOpts := &repoOptions{ + DataDir: dataDir, + ExtraRepos: extraRepos, + } + img, err := getOneImage(distroStr, imgTypeStr, archStr, repoOpts) if err != nil { return nil, err } @@ -146,7 +158,7 @@ func cmdManifestWrapper(pbar progress.ProgressBar, cmd *cobra.Command, args []st RpmDownloader: rpmDownloader, WithSBOM: withSBOM, } - err = generateManifest(dataDir, img, w, opts) + err = generateManifest(dataDir, extraRepos, img, w, opts) return img, err } @@ -276,7 +288,7 @@ func cmdDescribeImg(cmd *cobra.Command, args []string) error { archStr = arch.Current().String() } imgTypeStr := args[0] - res, err := getOneImage(dataDir, distroStr, imgTypeStr, archStr) + res, err := getOneImage(distroStr, imgTypeStr, archStr, &repoOptions{DataDir: dataDir}) if err != nil { return err } @@ -304,6 +316,7 @@ operating systems like Fedora, CentOS and RHEL with easy customizations support. SilenceErrors: true, } rootCmd.PersistentFlags().String("datadir", "", `Override the default data directory for e.g. custom repositories/*.json data`) + rootCmd.PersistentFlags().StringArray("extra-repo", nil, `Add an extra repository during build (will *not* be gpg checked and not be part of the final image)`) rootCmd.PersistentFlags().String("output-dir", "", `Put output into the specified directory`) rootCmd.PersistentFlags().BoolP("verbose", "v", false, `Switch to verbose mode`) rootCmd.SetOut(osStdout) diff --git a/cmd/image-builder/main_test.go b/cmd/image-builder/main_test.go index 3386bf8..95af02d 100644 --- a/cmd/image-builder/main_test.go +++ b/cmd/image-builder/main_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "os" + "os/exec" "path/filepath" "slices" "strings" @@ -15,6 +16,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" testrepos "github.com/osbuild/images/test/data/repositories" @@ -599,3 +601,48 @@ func TestProgressFromCmd(t *testing.T) { assert.Equal(t, tc.expectedProgress, fmt.Sprintf("%T", pbar)) } } + +func TestManifestExtraRepo(t *testing.T) { + if testing.Short() { + t.Skip("manifest generation takes a while") + } + if !hasDepsolveDnf() { + t.Skip("no osbuild-depsolve-dnf binary found") + } + if _, err := exec.LookPath("createrepo_c"); err != nil { + t.Skip("need createrepo_c to run this test") + } + + localRepoDir := filepath.Join(t.TempDir(), "repo") + err := os.MkdirAll(localRepoDir, 0755) + assert.NoError(t, err) + err = exec.Command("cp", "-a", "../../test/data/rpm/dummy-1.0.0-0.noarch.rpm", localRepoDir).Run() + assert.NoError(t, err) + err = exec.Command("createrepo_c", localRepoDir).Run() + assert.NoError(t, err) + + pkgHelloBlueprint := `{ + "packages": [ + {"name":"dummy"} + ] + }` + restore := main.MockOsArgs([]string{ + "manifest", + "qcow2", + "--distro=centos-9", + fmt.Sprintf("--extra-repo=file://%s", localRepoDir), + "--blueprint", makeTestBlueprint(t, pkgHelloBlueprint), + }) + defer restore() + + var fakeStdout bytes.Buffer + restore = main.MockOsStdout(&fakeStdout) + defer restore() + + err = main.Run() + require.NoError(t, err) + + // our local repo got added + assert.Contains(t, fakeStdout.String(), `"path":"dummy-1.0.0-0.noarch.rpm"`) + assert.Contains(t, fakeStdout.String(), fmt.Sprintf(`"url":"file://%s"`, localRepoDir)) +} diff --git a/cmd/image-builder/manifest.go b/cmd/image-builder/manifest.go index eed1cee..18bf5e6 100644 --- a/cmd/image-builder/manifest.go +++ b/cmd/image-builder/manifest.go @@ -39,8 +39,8 @@ func sbomWriter(outputDir, filename string, content io.Reader) error { return nil } -func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error { - repos, err := newRepoRegistry(dataDir) +func generateManifest(dataDir string, extraRepos []string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error { + repos, err := newRepoRegistry(dataDir, extraRepos) if err != nil { return err } diff --git a/cmd/image-builder/repos.go b/cmd/image-builder/repos.go index 4a2122c..48fb293 100644 --- a/cmd/image-builder/repos.go +++ b/cmd/image-builder/repos.go @@ -1,10 +1,13 @@ package main import ( + "fmt" "io/fs" + "net/url" "github.com/osbuild/images/data/repositories" "github.com/osbuild/images/pkg/reporegistry" + "github.com/osbuild/images/pkg/rpmmd" ) // defaultDataDirs contains the default search paths to look for @@ -16,7 +19,41 @@ var defaultDataDirs = []string{ "/usr/share/image-builder/repositories", } -var newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) { +type repoConfig struct { + DataDir string + ExtraRepos []string +} + +func parseExtraRepo(extraRepo string) ([]rpmmd.RepoConfig, error) { + // We want to eventually support more URIs repos here: + // - config:/path/to/repo.json + // - copr:@osbuild/osbuild (with full gpg retrival via the copr API) + // But for now just default to base-urls + + baseURL, err := url.Parse(extraRepo) + if err != nil { + return nil, fmt.Errorf("cannot parse extra repo %w", err) + } + if baseURL.Scheme == "" { + return nil, fmt.Errorf(`scheme missing in %q, please prefix with e.g. file:`, extraRepo) + } + + // TODO: to support gpg checking we will need to add signing keys. + // We will eventually add support for our own "repo.json" format + // which is rich enough to contain gpg keys (and more). + checkGPG := false + return []rpmmd.RepoConfig{ + { + Id: baseURL.String(), + Name: baseURL.String(), + BaseURLs: []string{baseURL.String()}, + CheckGPG: &checkGPG, + CheckRepoGPG: &checkGPG, + }, + }, nil +} + +var newRepoRegistry = func(dataDir string, extraRepos []string) (*reporegistry.RepoRegistry, error) { var dataDirs []string if dataDir != "" { dataDirs = []string{dataDir} @@ -24,5 +61,29 @@ var newRepoRegistry = func(dataDir string) (*reporegistry.RepoRegistry, error) { dataDirs = defaultDataDirs } - return reporegistry.New(dataDirs, []fs.FS{repos.FS}) + conf, err := reporegistry.LoadAllRepositories(dataDirs, []fs.FS{repos.FS}) + if err != nil { + return nil, err + } + + // XXX: this should probably go into manifestgen.Options as + // a new Options.ExtraRepoConf eventually (just like OverrideRepos) + for _, repo := range extraRepos { + // XXX: this loads the extra repo unconditionally to all + // distro/arch versions. we do not know in advance where + // it belongs to + extraRepo, err := parseExtraRepo(repo) + if err != nil { + return nil, err + } + for _, repoArchConfigs := range conf { + for arch := range repoArchConfigs { + archCfg := repoArchConfigs[arch] + archCfg = append(archCfg, extraRepo...) + repoArchConfigs[arch] = archCfg + } + } + } + + return reporegistry.NewFromDistrosRepoConfigs(conf), nil } diff --git a/cmd/image-builder/repos_test.go b/cmd/image-builder/repos_test.go new file mode 100644 index 0000000..6a7ab49 --- /dev/null +++ b/cmd/image-builder/repos_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/osbuild/images/pkg/rpmmd" +) + +func TestParseExtraRepoHappy(t *testing.T) { + checkGPG := false + + cfg, err := parseExtraRepo("file:///path/to/repo") + assert.NoError(t, err) + assert.Equal(t, cfg, []rpmmd.RepoConfig{ + { + Id: "file:///path/to/repo", + Name: "file:///path/to/repo", + BaseURLs: []string{"file:///path/to/repo"}, + CheckGPG: &checkGPG, + CheckRepoGPG: &checkGPG, + }, + }) +} + +func TestParseExtraRepoSad(t *testing.T) { + _, err := parseExtraRepo("/just/a/path") + assert.EqualError(t, err, `scheme missing in "/just/a/path", please prefix with e.g. file:`) +} diff --git a/test/data/rpm/build-dummy-rpm.sh b/test/data/rpm/build-dummy-rpm.sh new file mode 100755 index 0000000..d1db626 --- /dev/null +++ b/test/data/rpm/build-dummy-rpm.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +# inspired by from https://github.com/osbuild/osbuild-composer/blob/bdd2014c4467e9325b29624439c98584addc681f/test/cases/api.sh#L230-L262 +# thanks Thozza! + +# make a dummy rpm for our tests +DUMMYRPMDIR=$(mktemp -d) +DUMMYSPECFILE="$DUMMYRPMDIR/dummy.spec" +pushd "$DUMMYRPMDIR" + +cat < "$DUMMYSPECFILE" +#----------- spec file starts --------------- +Name: dummy +Version: 1.0.0 +Release: 0 +BuildArch: noarch +Vendor: dummy +Summary: Provides %{name} +License: BSD +Provides: dummy + +%description +%{summary} + +%files +EOF + +mkdir -p "DUMMYRPMDIR/rpmbuild" +rpmbuild --quiet --define "_topdir $DUMMYRPMDIR/rpmbuild" -bb "$DUMMYSPECFILE" +popd + +echo "Done building dummy rpm in $DUMMYRPMDIR" +cp "$DUMMYRPMDIR"/rpmbuild/RPMS/noarch/dummy-*.noarch.rpm . diff --git a/test/data/rpm/dummy-1.0.0-0.noarch.rpm b/test/data/rpm/dummy-1.0.0-0.noarch.rpm new file mode 100644 index 0000000000000000000000000000000000000000..a743c4ee20fb8024b6a6bbf2654a95130b2343a2 GIT binary patch literal 5565 zcmeHL&x;&I7_G^SaTCn462WNDOwd5~Fhg}!b$2yLG(QL)BrZxoPob)-YHgU=8D@5u zL~_Vs@vuS&tY=T&qMrN*Jm|r@UPOfuA(12|h~mLyeBX41gxoya1z*2@^?hGe*X!=f zyM6V}k8OrAa~s3qg+XXsV!V-l8~CL6t1v7 z5g9K_TNo*A67o_S9Rka^;KE8RoGoPHa;Nhkq_*0K#H&;rV@<}j7eZt~@id^)sJ3#VPc;w`>(E2L!dqB$T$o~LR{8bnMG2RNk>cwYG7dt&Pm2{Gcv*Xa@GDld?Y<4`XB9 zP7lG@-iLwopt46#j)vojpH96yT@CyhGxXDC7IQRns#^8a^+$6n=E7iP=Ec)D&8ALpC|GuV-Y^;N)~oSy-gh5GO?D|>H?Zf}*9<7&iKKdi>Ai09~S`ZYJ2 z^y;%w&&GP_X=Z{fd{QQb2`bmdhum;!eW86Y1w2AygwVkjp^#o`C0yomn<$g0EHx@E z=8v!-q~l5mtAaNu&-29VT;wXTR^~h}xRs^jsnp&CtFi<>q1S?2t!$bbLElgPQ_p7i z@8W*x(8qYc7`xh%V*#$Zi%VA*UwZZH1BZ`a{`A