diff --git a/cmd/image-builder/manifest.go b/cmd/image-builder/manifest.go index 7a4ab0c..8b89f4b 100644 --- a/cmd/image-builder/manifest.go +++ b/cmd/image-builder/manifest.go @@ -7,11 +7,12 @@ import ( "github.com/osbuild/images/pkg/distro" "github.com/osbuild/images/pkg/imagefilter" + "github.com/osbuild/images/pkg/manifestgen" "github.com/osbuild/images/pkg/osbuild" "github.com/osbuild/images/pkg/ostree" + "github.com/osbuild/images/pkg/sbom" "github.com/osbuild/image-builder-cli/internal/blueprintload" - "github.com/osbuild/image-builder-cli/internal/manifestgen" ) type manifestOptions struct { @@ -53,7 +54,7 @@ func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer, if err := os.MkdirAll(outputDir, 0755); err != nil { return err } - manifestGenOpts.SBOMWriter = func(filename string, content io.Reader) error { + manifestGenOpts.SBOMWriter = func(filename string, content io.Reader, docType sbom.StandardType) error { return sbomWriter(outputDir, filename, content) } } diff --git a/internal/manifestgen/manifestgen.go b/internal/manifestgen/manifestgen.go deleted file mode 100644 index 1e8d1c1..0000000 --- a/internal/manifestgen/manifestgen.go +++ /dev/null @@ -1,241 +0,0 @@ -package manifestgen - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "os" - "slices" - "strings" - - "github.com/osbuild/images/pkg/blueprint" - "github.com/osbuild/images/pkg/container" - "github.com/osbuild/images/pkg/distro" - "github.com/osbuild/images/pkg/dnfjson" - "github.com/osbuild/images/pkg/manifest" - "github.com/osbuild/images/pkg/osbuild" - "github.com/osbuild/images/pkg/ostree" - "github.com/osbuild/images/pkg/reporegistry" - "github.com/osbuild/images/pkg/rpmmd" - "github.com/osbuild/images/pkg/sbom" -) - -// XXX: all of the helpers below are duplicated from -// cmd/build/main.go:depsolve (and probably more places) should go -// into a common helper in "images" or images should do this on its -// own -func defaultDepsolver(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error) { - if cacheDir == "" { - var err error - cacheDir, err = os.MkdirTemp("", "manifestgen") - if err != nil { - return nil, fmt.Errorf("cannot create temporary directory: %w", err) - } - defer os.RemoveAll(cacheDir) - } - - solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) - depsolvedSets := make(map[string]dnfjson.DepsolveResult) - for name, pkgSet := range packageSets { - // XXX: is there harm in always generating an sbom? - // (expect for slightly longer runtime?) - res, err := solver.Depsolve(pkgSet, sbom.StandardTypeSpdx) - if err != nil { - return nil, fmt.Errorf("error depsolving: %w", err) - } - depsolvedSets[name] = *res - } - return depsolvedSets, nil -} - -func resolveContainers(containers []container.SourceSpec, archName string) ([]container.Spec, error) { - resolver := container.NewResolver(archName) - - for _, c := range containers { - resolver.Add(c) - } - - return resolver.Finish() -} - -func defaultContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { - containerSpecs := make(map[string][]container.Spec, len(containerSources)) - for plName, sourceSpecs := range containerSources { - specs, err := resolveContainers(sourceSpecs, archName) - if err != nil { - return nil, fmt.Errorf("error container resolving: %w", err) - } - containerSpecs[plName] = specs - } - return containerSpecs, nil -} - -func defaultCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { - commits := make(map[string][]ostree.CommitSpec, len(commitSources)) - for name, commitSources := range commitSources { - commitSpecs := make([]ostree.CommitSpec, len(commitSources)) - for idx, commitSource := range commitSources { - var err error - commitSpecs[idx], err = ostree.Resolve(commitSource) - if err != nil { - return nil, fmt.Errorf("error ostree commit resolving: %w", err) - } - } - commits[name] = commitSpecs - } - return commits, nil -} - -type ( - DepsolveFunc func(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error) - - ContainerResolverFunc func(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) - - CommitResolverFunc func(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) - - SBOMWriterFunc func(filename string, content io.Reader) error -) - -// Options contains the optional settings for the manifest generation. -// For unset values defaults will be used. -type Options struct { - Cachedir string - Output io.Writer - - // There are two types of sbom outputs, one for the "payload" - // and one for the "buildroot", we allow exporting both here - SbomImageOutput io.Writer - SbomBuildrootOutput io.Writer - - Depsolver DepsolveFunc - ContainerResolver ContainerResolverFunc - CommitResolver CommitResolverFunc - - RpmDownloader osbuild.RpmDownloader - - // Will be called for each generated SBOM the filename - // contains the suggest filename string and the content - // can be read - SBOMWriter SBOMWriterFunc -} - -// Generator can generate an osbuild manifest from a given repository -// and options. -type Generator struct { - cacheDir string - out io.Writer - - depsolver DepsolveFunc - containerResolver ContainerResolverFunc - commitResolver CommitResolverFunc - sbomWriter SBOMWriterFunc - - reporegistry *reporegistry.RepoRegistry - - rpmDownloader osbuild.RpmDownloader -} - -// New will create a new manifest generator -func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, error) { - if opts == nil { - opts = &Options{} - } - mg := &Generator{ - reporegistry: reporegistry, - - cacheDir: opts.Cachedir, - out: opts.Output, - depsolver: opts.Depsolver, - containerResolver: opts.ContainerResolver, - commitResolver: opts.CommitResolver, - rpmDownloader: opts.RpmDownloader, - sbomWriter: opts.SBOMWriter, - } - if mg.out == nil { - mg.out = os.Stdout - } - if mg.depsolver == nil { - mg.depsolver = defaultDepsolver - } - if mg.containerResolver == nil { - mg.containerResolver = defaultContainerResolver - } - if mg.commitResolver == nil { - mg.commitResolver = defaultCommitResolver - } - - return mg, nil -} - -// Generate will generate a new manifest for the given distro/imageType/arch -// combination. -func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgType distro.ImageType, a distro.Arch, imgOpts *distro.ImageOptions) error { - if imgOpts == nil { - imgOpts = &distro.ImageOptions{} - } - - repos, err := mg.reporegistry.ReposByImageTypeName(dist.Name(), a.Name(), imgType.Name()) - if err != nil { - return err - } - preManifest, warnings, err := imgType.Manifest(bp, *imgOpts, repos, nil) - if err != nil { - return err - } - if len(warnings) > 0 { - // XXX: what can we do here? for things like json output? - // what are these warnings? - return fmt.Errorf("warnings during manifest creation: %v", strings.Join(warnings, "\n")) - } - depsolved, err := mg.depsolver(mg.cacheDir, preManifest.GetPackageSetChains(), dist, a.Name()) - if err != nil { - return err - } - containerSpecs, err := mg.containerResolver(preManifest.GetContainerSourceSpecs(), a.Name()) - if err != nil { - return err - } - commitSpecs, err := mg.commitResolver(preManifest.GetOSTreeSourceSpecs()) - if err != nil { - return err - } - - opts := &manifest.SerializeOptions{ - RpmDownloader: mg.rpmDownloader, - } - mf, err := preManifest.Serialize(depsolved, containerSpecs, commitSpecs, opts) - if err != nil { - return err - } - fmt.Fprintf(mg.out, "%s\n", mf) - - if mg.sbomWriter != nil { - // XXX: this is very similar to - // osbuild-composer:jobimpl-osbuild.go, see if code - // can be shared - for plName, depsolvedPipeline := range depsolved { - pipelinePurpose := "unknown" - switch { - case slices.Contains(imgType.PayloadPipelines(), plName): - pipelinePurpose = "image" - case slices.Contains(imgType.BuildPipelines(), plName): - pipelinePurpose = "buildroot" - } - // XXX: sync with image-builder-cli:build.go name generation - can we have a shared helper? - imageName := fmt.Sprintf("%s-%s-%s", dist.Name(), imgType.Name(), a.Name()) - sbomDocOutputFilename := fmt.Sprintf("%s.%s-%s.spdx.json", imageName, pipelinePurpose, plName) - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - if err := enc.Encode(depsolvedPipeline.SBOM); err != nil { - return err - } - if err := mg.sbomWriter(sbomDocOutputFilename, &buf); err != nil { - return err - } - } - } - - return nil -} diff --git a/internal/manifestgen/manifestgen_test.go b/internal/manifestgen/manifestgen_test.go deleted file mode 100644 index e12d744..0000000 --- a/internal/manifestgen/manifestgen_test.go +++ /dev/null @@ -1,291 +0,0 @@ -package manifestgen_test - -import ( - "bytes" - "crypto/sha256" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "strings" - "testing" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/osbuild/images/pkg/blueprint" - "github.com/osbuild/images/pkg/container" - "github.com/osbuild/images/pkg/distro" - "github.com/osbuild/images/pkg/distrofactory" - "github.com/osbuild/images/pkg/dnfjson" - "github.com/osbuild/images/pkg/imagefilter" - "github.com/osbuild/images/pkg/osbuild" - "github.com/osbuild/images/pkg/ostree" - "github.com/osbuild/images/pkg/rpmmd" - "github.com/osbuild/images/pkg/sbom" - testrepos "github.com/osbuild/images/test/data/repositories" - - "github.com/osbuild/image-builder-cli/internal/manifestgen" - "github.com/osbuild/image-builder-cli/internal/manifesttest" -) - -func init() { - // silence logrus by default, it is quite verbose - logrus.SetLevel(logrus.WarnLevel) -} - -func sha256For(s string) string { - h := sha256.New() - h.Write([]byte(s)) - bs := h.Sum(nil) - return fmt.Sprintf("sha256:%x", bs) -} - -func TestManifestGeneratorDepsolve(t *testing.T) { - repos, err := testrepos.New() - assert.NoError(t, err) - fac := distrofactory.NewDefault() - - filter, err := imagefilter.New(fac, repos) - assert.NoError(t, err) - res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") - assert.NoError(t, err) - assert.Equal(t, 1, len(res)) - - for _, useLibrepo := range []bool{false, true} { - t.Run(fmt.Sprintf("useLibrepo: %v", useLibrepo), func(t *testing.T) { - var rpmDownloader osbuild.RpmDownloader - if useLibrepo { - rpmDownloader = osbuild.RpmDownloaderLibrepo - } - - var osbuildManifest bytes.Buffer - opts := &manifestgen.Options{ - Output: &osbuildManifest, - Depsolver: fakeDepsolve, - CommitResolver: panicCommitResolver, - ContainerResolver: panicContainerResolver, - - RpmDownloader: rpmDownloader, - } - mg, err := manifestgen.New(repos, opts) - assert.NoError(t, err) - assert.NotNil(t, mg) - var bp blueprint.Blueprint - err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) - require.NoError(t, err) - - pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes()) - assert.NoError(t, err) - assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames) - - // we expect at least a "kernel" package in the manifest, - // sadly the test distro does not really generate much here so we - // need to use this as a canary that resolving happend - // XXX: add testhelper to manifesttest for this - expectedSha256 := sha256For("kernel") - assert.Contains(t, osbuildManifest.String(), expectedSha256) - - assert.Equal(t, strings.Contains(osbuildManifest.String(), "org.osbuild.librepo"), useLibrepo) - }) - } -} - -func TestManifestGeneratorWithOstreeCommit(t *testing.T) { - var osbuildManifest bytes.Buffer - - repos, err := testrepos.New() - assert.NoError(t, err) - - fac := distrofactory.NewDefault() - filter, err := imagefilter.New(fac, repos) - assert.NoError(t, err) - res, err := filter.Filter("distro:centos-9", "type:edge-ami", "arch:x86_64") - assert.NoError(t, err) - assert.Equal(t, 1, len(res)) - - opts := &manifestgen.Options{ - Output: &osbuildManifest, - Depsolver: fakeDepsolve, - CommitResolver: fakeCommitResolver, - ContainerResolver: panicContainerResolver, - } - imageOpts := &distro.ImageOptions{ - OSTree: &ostree.ImageOptions{ - //ImageRef: "latest/1/x86_64/edge", - URL: "http://example.com/", - }, - } - mg, err := manifestgen.New(repos, opts) - assert.NoError(t, err) - assert.NotNil(t, mg) - var bp blueprint.Blueprint - err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, imageOpts) - assert.NoError(t, err) - - pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes()) - assert.NoError(t, err) - assert.Equal(t, []string{"build", "ostree-deployment", "image"}, pipelineNames) - - // XXX: add testhelper to manifesttest for this - assert.Contains(t, osbuildManifest.String(), `{"url":"resolved-url-for-centos/9/x86_64/edge"}`) - // we expect at least a "glibc" package in the manifest, - // sadly the test distro does not really generate much here so we - // need to use this as a canary that resolving happend - // XXX: add testhelper to manifesttest for this - expectedSha256 := sha256For("glibc") - assert.Contains(t, osbuildManifest.String(), expectedSha256) -} - -func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error) { - depsolvedSets := make(map[string]dnfjson.DepsolveResult) - for name, pkgSets := range packageSets { - repoId := fmt.Sprintf("repo_id_%s", name) - var resolvedSet dnfjson.DepsolveResult - for _, pkgSet := range pkgSets { - for _, pkgName := range pkgSet.Include { - resolvedSet.Packages = append(resolvedSet.Packages, rpmmd.PackageSpec{ - Name: pkgName, - Checksum: sha256For(pkgName), - Path: fmt.Sprintf("path/%s.rpm", pkgName), - RepoID: repoId, - }) - resolvedSet.Repos = append(resolvedSet.Repos, rpmmd.RepoConfig{ - Id: repoId, - Metalink: "https://example.com/metalink", - }) - doc, err := sbom.NewDocument(sbom.StandardTypeSpdx, json.RawMessage(fmt.Sprintf(`{"sbom-for":"%s"}`, name))) - if err != nil { - return nil, err - } - resolvedSet.SBOM = doc - } - } - depsolvedSets[name] = resolvedSet - } - return depsolvedSets, nil -} - -func fakeCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { - commits := make(map[string][]ostree.CommitSpec, len(commitSources)) - for name, commitSources := range commitSources { - commitSpecs := make([]ostree.CommitSpec, len(commitSources)) - for idx, commitSource := range commitSources { - commitSpecs[idx] = ostree.CommitSpec{ - URL: fmt.Sprintf("resolved-url-for-%s", commitSource.Ref), - } - } - commits[name] = commitSpecs - } - return commits, nil - -} - -func panicCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { - if len(commitSources) > 0 { - panic("panicCommitResolver") - } - return nil, nil -} - -func fakeContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { - containerSpecs := make(map[string][]container.Spec, len(containerSources)) - for plName, sourceSpecs := range containerSources { - var containers []container.Spec - for _, spec := range sourceSpecs { - containers = append(containers, container.Spec{ - Source: fmt.Sprintf("resolved-cnt-%s", spec.Source), - Digest: "sha256:" + sha256For("digest:"+spec.Source), - ImageID: "sha256:" + sha256For("id:"+spec.Source), - }) - } - containerSpecs[plName] = containers - } - return containerSpecs, nil -} - -func panicContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { - if len(containerSources) > 0 { - panic("panicContainerResolver") - } - return nil, nil -} - -func TestManifestGeneratorContainers(t *testing.T) { - repos, err := testrepos.New() - assert.NoError(t, err) - fac := distrofactory.NewDefault() - - filter, err := imagefilter.New(fac, repos) - assert.NoError(t, err) - res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") - assert.NoError(t, err) - assert.Equal(t, 1, len(res)) - - var osbuildManifest bytes.Buffer - opts := &manifestgen.Options{ - Output: &osbuildManifest, - Depsolver: fakeDepsolve, - CommitResolver: panicCommitResolver, - ContainerResolver: fakeContainerResolver, - } - mg, err := manifestgen.New(repos, opts) - assert.NoError(t, err) - assert.NotNil(t, mg) - fakeContainerSource := "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal" - bp := blueprint.Blueprint{ - Containers: []blueprint.Container{ - { - Source: fakeContainerSource, - }, - }, - } - err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) - assert.NoError(t, err) - - // container is included - assert.Contains(t, osbuildManifest.String(), "resolved-cnt-"+fakeContainerSource) -} - -func TestManifestGeneratorDepsolveWithSbomWriter(t *testing.T) { - repos, err := testrepos.New() - assert.NoError(t, err) - fac := distrofactory.NewDefault() - - filter, err := imagefilter.New(fac, repos) - assert.NoError(t, err) - res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") - assert.NoError(t, err) - assert.Equal(t, 1, len(res)) - - var osbuildManifest bytes.Buffer - generatedSboms := map[string]string{} - opts := &manifestgen.Options{ - Output: &osbuildManifest, - Depsolver: fakeDepsolve, - CommitResolver: panicCommitResolver, - ContainerResolver: panicContainerResolver, - - SBOMWriter: func(filename string, content io.Reader) error { - b, err := ioutil.ReadAll(content) - assert.NoError(t, err) - generatedSboms[filename] = strings.TrimSpace(string(b)) - return nil - }, - } - mg, err := manifestgen.New(repos, opts) - assert.NoError(t, err) - assert.NotNil(t, mg) - var bp blueprint.Blueprint - err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) - require.NoError(t, err) - - assert.Contains(t, generatedSboms, "centos-9-qcow2-x86_64.buildroot-build.spdx.json") - assert.Contains(t, generatedSboms, "centos-9-qcow2-x86_64.image-os.spdx.json") - expected := map[string]string{ - "centos-9-qcow2-x86_64.buildroot-build.spdx.json": `{"DocType":"spdx","Document":{"sbom-for":"build"}}`, - "centos-9-qcow2-x86_64.image-os.spdx.json": `{"DocType":"spdx","Document":{"sbom-for":"os"}}`, - } - assert.Equal(t, expected, generatedSboms) -} diff --git a/test/test_container.py b/test/test_container.py index 313b973..6c3f0e4 100644 --- a/test/test_container.py +++ b/test/test_container.py @@ -51,4 +51,4 @@ def test_container_manifest_generates_sbom(tmp_path, build_container): assert buildroot_sbom_json_path.exists() sbom_json = json.loads(image_sbom_json_path.read_text()) # smoke test that we have glibc in the json doc - assert "glibc" in [s["name"] for s in sbom_json["Document"]["packages"]] + assert "glibc" in [s["name"] for s in sbom_json["packages"]], f"missing glibc in {sbom_json}"