particle-os/debian-forge-cli
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

main: add experimental --use-librepo to support librepo downloads

Browse source 
This commit switches to the librepo enabled `images` library via:
```
go mod -replace github.com/osbuild/iamges=github.com/mvo5/images@librepo-sources-osbuild1974
```
which in turn needs osbuild PR#1974.

With that it then adds a new `--use-librepo` switch that will
enable librepo based downloading so that people can play with
the new backend.
This commit is contained in:
Michael Vogt 2025-01-07 12:37:35 +01:00 • committed by Simon de Vlieger
parent 78f62e21e2
commit f8ffa8a258
5 changed files with 96 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

12
cmd/image-builder/main.go
View file

@ -11,6 +11,7 @@ import (
"github.com/osbuild/images/pkg/arch"
"github.com/osbuild/images/pkg/imagefilter"
"github.com/osbuild/images/pkg/osbuild"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/image-builder-cli/internal/blueprintload"
@ -83,6 +84,14 @@ func cmdManifestWrapper(cmd *cobra.Command, args []string, w io.Writer, archChec
if err != nil {
return nil, err
}
useLibrepo, err := cmd.Flags().GetBool("use-librepo")
if err != nil {
return nil, err
}
var rpmDownloader osbuild.RpmDownloader
if useLibrepo {
rpmDownloader = osbuild.RpmDownloaderLibrepo
}
blueprintPath, err := cmd.Flags().GetString("blueprint")
if err != nil {
@ -111,7 +120,7 @@ func cmdManifestWrapper(cmd *cobra.Command, args []string, w io.Writer, archChec
}
}
err = generateManifest(dataDir, blueprintPath, res, w, ostreeImgOpts)
err = generateManifest(dataDir, blueprintPath, res, w, ostreeImgOpts, rpmDownloader)
return res, err
}
@ -185,6 +194,7 @@ operating sytsems like centos and RHEL with easy customizations support.`,
manifestCmd.Flags().String("ostree-ref", "", `OSTREE reference`)
manifestCmd.Flags().String("ostree-parent", "", `OSTREE parent`)
manifestCmd.Flags().String("ostree-url", "", `OSTREE url`)
manifestCmd.Flags().Bool("use-librepo", false, `(experimental) use librepo to download packages, needs new osbuild`)
rootCmd.AddCommand(manifestCmd)
buildCmd := &cobra.Command{

45
cmd/image-builder/main_test.go
View file

@ -162,29 +162,36 @@ func TestManifestIntegrationSmoke(t *testing.T) {
restore := main.MockNewRepoRegistry(testrepos.New)
defer restore()
restore = main.MockOsArgs([]string{
"manifest",
"qcow2",
"--arch=x86_64",
"--distro=centos-9",
fmt.Sprintf("--blueprint=%s", makeTestBlueprint(t, testBlueprint)),
})
defer restore()
for _, useLibrepo := range []bool{false, true} {
t.Run(fmt.Sprintf("use-librepo: %v", useLibrepo), func(t *testing.T) {
restore = main.MockOsArgs([]string{
"manifest",
"qcow2",
"--arch=x86_64",
"--distro=centos-9",
fmt.Sprintf("--blueprint=%s", makeTestBlueprint(t, testBlueprint)),
fmt.Sprintf("--use-librepo=%v", useLibrepo),
})
defer restore()
var fakeStdout bytes.Buffer
restore = main.MockOsStdout(&fakeStdout)
defer restore()
var fakeStdout bytes.Buffer
restore = main.MockOsStdout(&fakeStdout)
defer restore()
err := main.Run()
assert.NoError(t, err)
err := main.Run()
assert.NoError(t, err)
pipelineNames, err := manifesttest.PipelineNamesFrom(fakeStdout.Bytes())
assert.NoError(t, err)
assert.Contains(t, pipelineNames, "qcow2")
pipelineNames, err := manifesttest.PipelineNamesFrom(fakeStdout.Bytes())
assert.NoError(t, err)
assert.Contains(t, pipelineNames, "qcow2")
// XXX: provide helpers in manifesttest to extract this in a nicer way
assert.Contains(t, fakeStdout.String(), `{"type":"org.osbuild.users","options":{"users":{"alice":{}}}}`)
assert.Contains(t, fakeStdout.String(), `"image":{"name":"registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal"`)
// XXX: provide helpers in manifesttest to extract this in a nicer way
assert.Contains(t, fakeStdout.String(), `{"type":"org.osbuild.users","options":{"users":{"alice":{}}}}`)
assert.Contains(t, fakeStdout.String(), `"image":{"name":"registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal"`)
assert.Equal(t, strings.Contains(fakeStdout.String(), "org.osbuild.librepo"), useLibrepo)
})
}
}
func TestManifestIntegrationCrossArch(t *testing.T) {

6
cmd/image-builder/manifest.go
View file

@ -5,20 +5,22 @@ import (
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/imagefilter"
"github.com/osbuild/images/pkg/osbuild"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/image-builder-cli/internal/blueprintload"
"github.com/osbuild/image-builder-cli/internal/manifestgen"
)
func generateManifest(dataDir, blueprintPath string, res *imagefilter.Result, output io.Writer, ostreeOpts *ostree.ImageOptions) error {
func generateManifest(dataDir, blueprintPath string, res *imagefilter.Result, output io.Writer, ostreeOpts *ostree.ImageOptions, rpmDownloader osbuild.RpmDownloader) error {
repos, err := newRepoRegistry(dataDir)
if err != nil {
return err
}
// XXX: add --rpmmd/cachedir option like bib
mg, err := manifestgen.New(repos, &manifestgen.Options{
Output: output,
Output: output,
RpmDownloader: rpmDownloader,
})
if err != nil {
return err

8
internal/manifestgen/manifestgen.go
View file

@ -10,6 +10,7 @@ import (
"github.com/osbuild/images/pkg/container"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/dnfjson"
"github.com/osbuild/images/pkg/osbuild"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/reporegistry"
"github.com/osbuild/images/pkg/rpmmd"
@ -100,6 +101,8 @@ type Options struct {
Depsolver DepsolveFunc
ContainerResolver ContainerResolverFunc
CommitResolver CommitResolverFunc
RpmDownloader osbuild.RpmDownloader
}
// Generator can generate an osbuild manifest from a given repository
@ -113,6 +116,8 @@ type Generator struct {
commitResolver CommitResolverFunc
reporegistry *reporegistry.RepoRegistry
rpmDownloader osbuild.RpmDownloader
}
// New will create a new manifest generator
@ -128,6 +133,7 @@ func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, er
depsolver: opts.Depsolver,
containerResolver: opts.ContainerResolver,
commitResolver: opts.CommitResolver,
rpmDownloader: opts.RpmDownloader,
}
if mg.out == nil {
mg.out = os.Stdout
@ -177,7 +183,7 @@ func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgTy
if err != nil {
return err
}
mf, err := preManifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfig)
mf, err := preManifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfig, mg.rpmDownloader)
if err != nil {
return err
}

71
internal/manifestgen/manifestgen_test.go
View file

@ -4,16 +4,19 @@ import (
"bytes"
"crypto/sha256"
"fmt"
"strings"
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/osbuild/images/pkg/blueprint"
"github.com/osbuild/images/pkg/container"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/distrofactory"
"github.com/osbuild/images/pkg/imagefilter"
"github.com/osbuild/images/pkg/osbuild"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/rpmmd"
testrepos "github.com/osbuild/images/test/data/repositories"
@ -45,30 +48,43 @@ func TestManifestGeneratorDepsolve(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 1, len(res))
var osbuildManifest bytes.Buffer
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
CommitResolver: panicCommitResolver,
ContainerResolver: panicContainerResolver,
for _, useLibrepo := range []bool{false, true} {
t.Run(fmt.Sprintf("useLibrepo: %v", useLibrepo), func(t *testing.T) {
var rpmDownloader osbuild.RpmDownloader
if useLibrepo {
rpmDownloader = osbuild.RpmDownloaderLibrepo
}
var osbuildManifest bytes.Buffer
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
CommitResolver: panicCommitResolver,
ContainerResolver: panicContainerResolver,
RpmDownloader: rpmDownloader,
}
mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)
assert.NotNil(t, mg)
var bp blueprint.Blueprint
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
require.NoError(t, err)
pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes())
assert.NoError(t, err)
assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames)
// we expect at least a "kernel" package in the manifest,
// sadly the test distro does not really generate much here so we
// need to use this as a canary that resolving happend
// XXX: add testhelper to manifesttest for this
expectedSha256 := sha256For("kernel")
assert.Contains(t, osbuildManifest.String(), expectedSha256)
assert.Equal(t, strings.Contains(osbuildManifest.String(), "org.osbuild.librepo"), useLibrepo)
})
}
mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)
assert.NotNil(t, mg)
var bp blueprint.Blueprint
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
assert.NoError(t, err)
pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes())
assert.NoError(t, err)
assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames)
// we expect at least a "kernel" package in the manifest,
// sadly the test distro does not really generate much here so we
// need to use this as a canary that resolving happend
// XXX: add testhelper to manifesttest for this
expectedSha256 := sha256For("kernel")
assert.Contains(t, osbuildManifest.String(), expectedSha256)
}
func TestManifestGeneratorWithOstreeCommit(t *testing.T) {
@ -121,16 +137,25 @@ func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
repoSets := make(map[string][]rpmmd.RepoConfig)
for name, pkgSets := range packageSets {
repoId := fmt.Sprintf("repo_id_%s", name)
var resolvedSet []rpmmd.PackageSpec
for _, pkgSet := range pkgSets {
for _, pkgName := range pkgSet.Include {
resolvedSet = append(resolvedSet, rpmmd.PackageSpec{
Name: pkgName,
Checksum: sha256For(pkgName),
Path: fmt.Sprintf("path/%s.rpm", pkgName),
RepoID: repoId,
})
}
}
depsolvedSets[name] = resolvedSet
repoSets[name] = []rpmmd.RepoConfig{
{
Id: repoId,
Metalink: "http://example.com/metalink",
},
}
}
return depsolvedSets, repoSets, nil
}