particle-os/debian-forge-cli
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

image-builder: drop internal manifestgen in favor of images

Browse source 
This commit drops the internal `manifestgen` package in favor
of using the version in `images` now that
https://github.com/osbuild/images/pull/1153 is merged.
This commit is contained in:
Michael Vogt 2025-01-24 16:03:18 +01:00 • committed by Simon de Vlieger
parent cf890c7297
commit 98e4bebcfa
4 changed files with 4 additions and 535 deletions
Download patch file Download diff file Expand all files Collapse all files

5
cmd/image-builder/manifest.go
View file

@ -7,11 +7,12 @@ import (
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/imagefilter"
"github.com/osbuild/images/pkg/manifestgen"
"github.com/osbuild/images/pkg/osbuild"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/sbom"
"github.com/osbuild/image-builder-cli/internal/blueprintload"
"github.com/osbuild/image-builder-cli/internal/manifestgen"
)
type manifestOptions struct {
@ -53,7 +54,7 @@ func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer,
if err := os.MkdirAll(outputDir, 0755); err != nil {
return err
}
manifestGenOpts.SBOMWriter = func(filename string, content io.Reader) error {
manifestGenOpts.SBOMWriter = func(filename string, content io.Reader, docType sbom.StandardType) error {
return sbomWriter(outputDir, filename, content)
}
}

241
internal/manifestgen/manifestgen.go
View file

@ -1,241 +0,0 @@
package manifestgen
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"slices"
"strings"
"github.com/osbuild/images/pkg/blueprint"
"github.com/osbuild/images/pkg/container"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/dnfjson"
"github.com/osbuild/images/pkg/manifest"
"github.com/osbuild/images/pkg/osbuild"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/reporegistry"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/images/pkg/sbom"
)
// XXX: all of the helpers below are duplicated from
// cmd/build/main.go:depsolve (and probably more places) should go
// into a common helper in "images" or images should do this on its
// own
func defaultDepsolver(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error) {
if cacheDir == "" {
var err error
cacheDir, err = os.MkdirTemp("", "manifestgen")
if err != nil {
return nil, fmt.Errorf("cannot create temporary directory: %w", err)
}
defer os.RemoveAll(cacheDir)
}
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir)
depsolvedSets := make(map[string]dnfjson.DepsolveResult)
for name, pkgSet := range packageSets {
// XXX: is there harm in always generating an sbom?
// (expect for slightly longer runtime?)
res, err := solver.Depsolve(pkgSet, sbom.StandardTypeSpdx)
if err != nil {
return nil, fmt.Errorf("error depsolving: %w", err)
}
depsolvedSets[name] = *res
}
return depsolvedSets, nil
}
func resolveContainers(containers []container.SourceSpec, archName string) ([]container.Spec, error) {
resolver := container.NewResolver(archName)
for _, c := range containers {
resolver.Add(c)
}
return resolver.Finish()
}
func defaultContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) {
containerSpecs := make(map[string][]container.Spec, len(containerSources))
for plName, sourceSpecs := range containerSources {
specs, err := resolveContainers(sourceSpecs, archName)
if err != nil {
return nil, fmt.Errorf("error container resolving: %w", err)
}
containerSpecs[plName] = specs
}
return containerSpecs, nil
}
func defaultCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) {
commits := make(map[string][]ostree.CommitSpec, len(commitSources))
for name, commitSources := range commitSources {
commitSpecs := make([]ostree.CommitSpec, len(commitSources))
for idx, commitSource := range commitSources {
var err error
commitSpecs[idx], err = ostree.Resolve(commitSource)
if err != nil {
return nil, fmt.Errorf("error ostree commit resolving: %w", err)
}
}
commits[name] = commitSpecs
}
return commits, nil
}
type (
DepsolveFunc func(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error)
ContainerResolverFunc func(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error)
CommitResolverFunc func(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error)
SBOMWriterFunc func(filename string, content io.Reader) error
)
// Options contains the optional settings for the manifest generation.
// For unset values defaults will be used.
type Options struct {
Cachedir string
Output io.Writer
// There are two types of sbom outputs, one for the "payload"
// and one for the "buildroot", we allow exporting both here
SbomImageOutput io.Writer
SbomBuildrootOutput io.Writer
Depsolver DepsolveFunc
ContainerResolver ContainerResolverFunc
CommitResolver CommitResolverFunc
RpmDownloader osbuild.RpmDownloader
// Will be called for each generated SBOM the filename
// contains the suggest filename string and the content
// can be read
SBOMWriter SBOMWriterFunc
}
// Generator can generate an osbuild manifest from a given repository
// and options.
type Generator struct {
cacheDir string
out io.Writer
depsolver DepsolveFunc
containerResolver ContainerResolverFunc
commitResolver CommitResolverFunc
sbomWriter SBOMWriterFunc
reporegistry *reporegistry.RepoRegistry
rpmDownloader osbuild.RpmDownloader
}
// New will create a new manifest generator
func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, error) {
if opts == nil {
opts = &Options{}
}
mg := &Generator{
reporegistry: reporegistry,
cacheDir: opts.Cachedir,
out: opts.Output,
depsolver: opts.Depsolver,
containerResolver: opts.ContainerResolver,
commitResolver: opts.CommitResolver,
rpmDownloader: opts.RpmDownloader,
sbomWriter: opts.SBOMWriter,
}
if mg.out == nil {
mg.out = os.Stdout
}
if mg.depsolver == nil {
mg.depsolver = defaultDepsolver
}
if mg.containerResolver == nil {
mg.containerResolver = defaultContainerResolver
}
if mg.commitResolver == nil {
mg.commitResolver = defaultCommitResolver
}
return mg, nil
}
// Generate will generate a new manifest for the given distro/imageType/arch
// combination.
func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgType distro.ImageType, a distro.Arch, imgOpts *distro.ImageOptions) error {
if imgOpts == nil {
imgOpts = &distro.ImageOptions{}
}
repos, err := mg.reporegistry.ReposByImageTypeName(dist.Name(), a.Name(), imgType.Name())
if err != nil {
return err
}
preManifest, warnings, err := imgType.Manifest(bp, *imgOpts, repos, nil)
if err != nil {
return err
}
if len(warnings) > 0 {
// XXX: what can we do here? for things like json output?
// what are these warnings?
return fmt.Errorf("warnings during manifest creation: %v", strings.Join(warnings, "\n"))
}
depsolved, err := mg.depsolver(mg.cacheDir, preManifest.GetPackageSetChains(), dist, a.Name())
if err != nil {
return err
}
containerSpecs, err := mg.containerResolver(preManifest.GetContainerSourceSpecs(), a.Name())
if err != nil {
return err
}
commitSpecs, err := mg.commitResolver(preManifest.GetOSTreeSourceSpecs())
if err != nil {
return err
}
opts := &manifest.SerializeOptions{
RpmDownloader: mg.rpmDownloader,
}
mf, err := preManifest.Serialize(depsolved, containerSpecs, commitSpecs, opts)
if err != nil {
return err
}
fmt.Fprintf(mg.out, "%s\n", mf)
if mg.sbomWriter != nil {
// XXX: this is very similar to
// osbuild-composer:jobimpl-osbuild.go, see if code
// can be shared
for plName, depsolvedPipeline := range depsolved {
pipelinePurpose := "unknown"
switch {
case slices.Contains(imgType.PayloadPipelines(), plName):
pipelinePurpose = "image"
case slices.Contains(imgType.BuildPipelines(), plName):
pipelinePurpose = "buildroot"
}
// XXX: sync with image-builder-cli:build.go name generation - can we have a shared helper?
imageName := fmt.Sprintf("%s-%s-%s", dist.Name(), imgType.Name(), a.Name())
sbomDocOutputFilename := fmt.Sprintf("%s.%s-%s.spdx.json", imageName, pipelinePurpose, plName)
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
if err := enc.Encode(depsolvedPipeline.SBOM); err != nil {
return err
}
if err := mg.sbomWriter(sbomDocOutputFilename, &buf); err != nil {
return err
}
}
}
return nil
}

291
internal/manifestgen/manifestgen_test.go
View file

@ -1,291 +0,0 @@
package manifestgen_test
import (
"bytes"
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"strings"
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/osbuild/images/pkg/blueprint"
"github.com/osbuild/images/pkg/container"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/distrofactory"
"github.com/osbuild/images/pkg/dnfjson"
"github.com/osbuild/images/pkg/imagefilter"
"github.com/osbuild/images/pkg/osbuild"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/images/pkg/sbom"
testrepos "github.com/osbuild/images/test/data/repositories"
"github.com/osbuild/image-builder-cli/internal/manifestgen"
"github.com/osbuild/image-builder-cli/internal/manifesttest"
)
func init() {
// silence logrus by default, it is quite verbose
logrus.SetLevel(logrus.WarnLevel)
}
func sha256For(s string) string {
h := sha256.New()
h.Write([]byte(s))
bs := h.Sum(nil)
return fmt.Sprintf("sha256:%x", bs)
}
func TestManifestGeneratorDepsolve(t *testing.T) {
repos, err := testrepos.New()
assert.NoError(t, err)
fac := distrofactory.NewDefault()
filter, err := imagefilter.New(fac, repos)
assert.NoError(t, err)
res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64")
assert.NoError(t, err)
assert.Equal(t, 1, len(res))
for _, useLibrepo := range []bool{false, true} {
t.Run(fmt.Sprintf("useLibrepo: %v", useLibrepo), func(t *testing.T) {
var rpmDownloader osbuild.RpmDownloader
if useLibrepo {
rpmDownloader = osbuild.RpmDownloaderLibrepo
}
var osbuildManifest bytes.Buffer
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
CommitResolver: panicCommitResolver,
ContainerResolver: panicContainerResolver,
RpmDownloader: rpmDownloader,
}
mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)
assert.NotNil(t, mg)
var bp blueprint.Blueprint
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
require.NoError(t, err)
pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes())
assert.NoError(t, err)
assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames)
// we expect at least a "kernel" package in the manifest,
// sadly the test distro does not really generate much here so we
// need to use this as a canary that resolving happend
// XXX: add testhelper to manifesttest for this
expectedSha256 := sha256For("kernel")
assert.Contains(t, osbuildManifest.String(), expectedSha256)
assert.Equal(t, strings.Contains(osbuildManifest.String(), "org.osbuild.librepo"), useLibrepo)
})
}
}
func TestManifestGeneratorWithOstreeCommit(t *testing.T) {
var osbuildManifest bytes.Buffer
repos, err := testrepos.New()
assert.NoError(t, err)
fac := distrofactory.NewDefault()
filter, err := imagefilter.New(fac, repos)
assert.NoError(t, err)
res, err := filter.Filter("distro:centos-9", "type:edge-ami", "arch:x86_64")
assert.NoError(t, err)
assert.Equal(t, 1, len(res))
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
CommitResolver: fakeCommitResolver,
ContainerResolver: panicContainerResolver,
}
imageOpts := &distro.ImageOptions{
OSTree: &ostree.ImageOptions{
//ImageRef: "latest/1/x86_64/edge",
URL: "http://example.com/",
},
}
mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)
assert.NotNil(t, mg)
var bp blueprint.Blueprint
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, imageOpts)
assert.NoError(t, err)
pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes())
assert.NoError(t, err)
assert.Equal(t, []string{"build", "ostree-deployment", "image"}, pipelineNames)
// XXX: add testhelper to manifesttest for this
assert.Contains(t, osbuildManifest.String(), `{"url":"resolved-url-for-centos/9/x86_64/edge"}`)
// we expect at least a "glibc" package in the manifest,
// sadly the test distro does not really generate much here so we
// need to use this as a canary that resolving happend
// XXX: add testhelper to manifesttest for this
expectedSha256 := sha256For("glibc")
assert.Contains(t, osbuildManifest.String(), expectedSha256)
}
func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error) {
depsolvedSets := make(map[string]dnfjson.DepsolveResult)
for name, pkgSets := range packageSets {
repoId := fmt.Sprintf("repo_id_%s", name)
var resolvedSet dnfjson.DepsolveResult
for _, pkgSet := range pkgSets {
for _, pkgName := range pkgSet.Include {
resolvedSet.Packages = append(resolvedSet.Packages, rpmmd.PackageSpec{
Name: pkgName,
Checksum: sha256For(pkgName),
Path: fmt.Sprintf("path/%s.rpm", pkgName),
RepoID: repoId,
})
resolvedSet.Repos = append(resolvedSet.Repos, rpmmd.RepoConfig{
Id: repoId,
Metalink: "https://example.com/metalink",
})
doc, err := sbom.NewDocument(sbom.StandardTypeSpdx, json.RawMessage(fmt.Sprintf(`{"sbom-for":"%s"}`, name)))
if err != nil {
return nil, err
}
resolvedSet.SBOM = doc
}
}
depsolvedSets[name] = resolvedSet
}
return depsolvedSets, nil
}
func fakeCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) {
commits := make(map[string][]ostree.CommitSpec, len(commitSources))
for name, commitSources := range commitSources {
commitSpecs := make([]ostree.CommitSpec, len(commitSources))
for idx, commitSource := range commitSources {
commitSpecs[idx] = ostree.CommitSpec{
URL: fmt.Sprintf("resolved-url-for-%s", commitSource.Ref),
}
}
commits[name] = commitSpecs
}
return commits, nil
}
func panicCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) {
if len(commitSources) > 0 {
panic("panicCommitResolver")
}
return nil, nil
}
func fakeContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) {
containerSpecs := make(map[string][]container.Spec, len(containerSources))
for plName, sourceSpecs := range containerSources {
var containers []container.Spec
for _, spec := range sourceSpecs {
containers = append(containers, container.Spec{
Source: fmt.Sprintf("resolved-cnt-%s", spec.Source),
Digest: "sha256:" + sha256For("digest:"+spec.Source),
ImageID: "sha256:" + sha256For("id:"+spec.Source),
})
}
containerSpecs[plName] = containers
}
return containerSpecs, nil
}
func panicContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) {
if len(containerSources) > 0 {
panic("panicContainerResolver")
}
return nil, nil
}
func TestManifestGeneratorContainers(t *testing.T) {
repos, err := testrepos.New()
assert.NoError(t, err)
fac := distrofactory.NewDefault()
filter, err := imagefilter.New(fac, repos)
assert.NoError(t, err)
res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64")
assert.NoError(t, err)
assert.Equal(t, 1, len(res))
var osbuildManifest bytes.Buffer
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
CommitResolver: panicCommitResolver,
ContainerResolver: fakeContainerResolver,
}
mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)
assert.NotNil(t, mg)
fakeContainerSource := "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal"
bp := blueprint.Blueprint{
Containers: []blueprint.Container{
{
Source: fakeContainerSource,
},
},
}
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
assert.NoError(t, err)
// container is included
assert.Contains(t, osbuildManifest.String(), "resolved-cnt-"+fakeContainerSource)
}
func TestManifestGeneratorDepsolveWithSbomWriter(t *testing.T) {
repos, err := testrepos.New()
assert.NoError(t, err)
fac := distrofactory.NewDefault()
filter, err := imagefilter.New(fac, repos)
assert.NoError(t, err)
res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64")
assert.NoError(t, err)
assert.Equal(t, 1, len(res))
var osbuildManifest bytes.Buffer
generatedSboms := map[string]string{}
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
CommitResolver: panicCommitResolver,
ContainerResolver: panicContainerResolver,
SBOMWriter: func(filename string, content io.Reader) error {
b, err := ioutil.ReadAll(content)
assert.NoError(t, err)
generatedSboms[filename] = strings.TrimSpace(string(b))
return nil
},
}
mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)
assert.NotNil(t, mg)
var bp blueprint.Blueprint
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
require.NoError(t, err)
assert.Contains(t, generatedSboms, "centos-9-qcow2-x86_64.buildroot-build.spdx.json")
assert.Contains(t, generatedSboms, "centos-9-qcow2-x86_64.image-os.spdx.json")
expected := map[string]string{
"centos-9-qcow2-x86_64.buildroot-build.spdx.json": `{"DocType":"spdx","Document":{"sbom-for":"build"}}`,
"centos-9-qcow2-x86_64.image-os.spdx.json": `{"DocType":"spdx","Document":{"sbom-for":"os"}}`,
}
assert.Equal(t, expected, generatedSboms)
}

2
test/test_container.py
View file

@ -51,4 +51,4 @@ def test_container_manifest_generates_sbom(tmp_path, build_container):
assert buildroot_sbom_json_path.exists()
sbom_json = json.loads(image_sbom_json_path.read_text())
# smoke test that we have glibc in the json doc
assert "glibc" in [s["name"] for s in sbom_json["Document"]["packages"]]
assert "glibc" in [s["name"] for s in sbom_json["packages"]], f"missing glibc in {sbom_json}"