ibcli: add new --extra-artifacts option with sbom support

This commit adds an option --extra-artifacts that can be
used to generate extra artifacts during the build or manifest
generation. Initially supported is `sbom` (but `manifest` is
planned too).

To use it run `--extra-artifacts=sbom` and it will generate
files like `centos-9-qcow2-x86_64.image-os.spdx.json` in
the output directory next to the generate runable artifact.

Closes: https://github.com/osbuild/image-builder-cli/issues/46

cmd/image-builder/build.go

@@ -1,9 +1,6 @@
 package main
 
 import (
-	"fmt"
-	"path/filepath"
-
 	"github.com/osbuild/images/pkg/imagefilter"
 	"github.com/osbuild/images/pkg/osbuild"
 )
@@ -12,9 +9,7 @@ func buildImage(res *imagefilter.Result, osbuildManifest []byte, osbuildStoreDir
 	// XXX: support output dir via commandline
 	// XXX2: support output filename via commandline (c.f.
 	//   https://github.com/osbuild/images/pull/1039)
-	outputDir := "."
+	jobOutputDir := outputDirFor(res)
-	buildName := fmt.Sprintf("%s-%s-%s", res.Distro.Name(), res.ImgType.Name(), res.Arch.Name())
-	jobOutputDir := filepath.Join(outputDir, buildName)
 
 	// XXX: support stremaing via images/pkg/osbuild/monitor.go
 	_, err := osbuild.RunOSBuild(osbuildManifest, osbuildStoreDir, jobOutputDir, res.ImgType.Exports(), nil, nil, false, osStderr)

cmd/image-builder/main.go

@@ -22,6 +22,11 @@ var (
 	osStderr io.Writer = os.Stderr
 )
 
+// generate the default output directory name for the given image
+func outputDirFor(img *imagefilter.Result) string {
+	return fmt.Sprintf("%s-%s-%s", img.Distro.Name(), img.ImgType.Name(), img.Arch.Name())
+}
+
 func cmdListImages(cmd *cobra.Command, args []string) error {
 	filter, err := cmd.Flags().GetStringArray("filter")
 	if err != nil {
@@ -80,6 +85,10 @@ func cmdManifestWrapper(cmd *cobra.Command, args []string, w io.Writer, archChec
 	if err != nil {
 		return nil, err
 	}
+	extraArtifacts, err := cmd.Flags().GetStringArray("extra-artifacts")
+	if err != nil {
+		return nil, err
+	}
 	ostreeImgOpts, err := ostreeImageOptions(cmd)
 	if err != nil {
 		return nil, err
@@ -121,9 +130,10 @@ func cmdManifestWrapper(cmd *cobra.Command, args []string, w io.Writer, archChec
 	}
 
 	opts := &manifestOptions{
-		BlueprintPath: blueprintPath,
+		BlueprintPath:  blueprintPath,
-		Ostree:        ostreeImgOpts,
+		Ostree:         ostreeImgOpts,
-		RpmDownloader: rpmDownloader,
+		RpmDownloader:  rpmDownloader,
+		ExtraArtifacts: extraArtifacts,
 	}
 	err = generateManifest(dataDir, img, w, opts)
 	return img, err
@@ -171,6 +181,7 @@ operating sytsems like centos and RHEL with easy customizations support.`,
 		SilenceErrors: true,
 	}
 	rootCmd.PersistentFlags().String("datadir", "", `Override the default data direcotry for e.g. custom repositories/*.json data`)
+	rootCmd.PersistentFlags().StringArray("extra-artifacts", nil, `Export extra artifacts to the output dir (e.g. "sbom")`)
 	rootCmd.SetOut(osStdout)
 	rootCmd.SetErr(osStderr)
 

cmd/image-builder/manifest.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"io"
+	"os"
+	"path/filepath"
+	"slices"
 
 	"github.com/osbuild/images/pkg/distro"
 	"github.com/osbuild/images/pkg/imagefilter"
@@ -13,9 +16,23 @@ import (
 )
 
 type manifestOptions struct {
-	BlueprintPath string
+	BlueprintPath  string
-	Ostree        *ostree.ImageOptions
+	Ostree         *ostree.ImageOptions
-	RpmDownloader osbuild.RpmDownloader
+	RpmDownloader  osbuild.RpmDownloader
+	ExtraArtifacts []string
+}
+
+func sbomWriter(outputDir, filename string, content io.Reader) error {
+	p := filepath.Join(outputDir, filename)
+	f, err := os.Create(p)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := io.Copy(f, content); err != nil {
+		return err
+	}
+	return nil
 }
 
 func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error {
@@ -24,13 +41,25 @@ func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer,
 		return err
 	}
 	// XXX: add --rpmmd/cachedir option like bib
-	mg, err := manifestgen.New(repos, &manifestgen.Options{
+	manifestGenOpts := &manifestgen.Options{
 		Output:        output,
 		RpmDownloader: opts.RpmDownloader,
-	})
+	}
+	if slices.Contains(opts.ExtraArtifacts, "sbom") {
+		outputDir := outputDirFor(img)
+		if err := os.MkdirAll(outputDir, 0755); err != nil {
+			return err
+		}
+		manifestGenOpts.SBOMWriter = func(filename string, content io.Reader) error {
+			return sbomWriter(outputDir, filename, content)
+		}
+	}
+
+	mg, err := manifestgen.New(repos, manifestGenOpts)
 	if err != nil {
 		return err
 	}
+
 	bp, err := blueprintload.Load(opts.BlueprintPath)
 	if err != nil {
 		return err

internal/manifestgen/manifestgen.go

@@ -1,9 +1,12 @@
 package manifestgen
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
+	"slices"
 	"strings"
 
 	"github.com/osbuild/images/pkg/blueprint"
@@ -35,7 +38,9 @@ func defaultDepsolver(cacheDir string, packageSets map[string][]rpmmd.PackageSet
 	solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir)
 	depsolvedSets := make(map[string]dnfjson.DepsolveResult)
 	for name, pkgSet := range packageSets {
-		res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone)
+		// XXX: is there harm in always generating an sbom?
+		// (expect for slightly longer runtime?)
+		res, err := solver.Depsolve(pkgSet, sbom.StandardTypeSpdx)
 		if err != nil {
 			return nil, fmt.Errorf("error depsolving: %w", err)
 		}
@@ -88,18 +93,31 @@ type (
 	ContainerResolverFunc func(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error)
 
 	CommitResolverFunc func(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error)
+
+	SBOMWriterFunc func(filename string, content io.Reader) error
 )
 
 // Options contains the optional settings for the manifest generation.
 // For unset values defaults will be used.
 type Options struct {
-	Cachedir          string
+	Cachedir string
-	Output            io.Writer
+	Output   io.Writer
+
+	// There are two types of sbom outputs, one for the "payload"
+	// and one for the "buildroot", we allow exporting both here
+	SbomImageOutput     io.Writer
+	SbomBuildrootOutput io.Writer
+
 	Depsolver         DepsolveFunc
 	ContainerResolver ContainerResolverFunc
 	CommitResolver    CommitResolverFunc
 
 	RpmDownloader osbuild.RpmDownloader
+
+	// Will be called for each generated SBOM the filename
+	// contains the suggest filename string and the content
+	// can be read
+	SBOMWriter SBOMWriterFunc
 }
 
 // Generator can generate an osbuild manifest from a given repository
@@ -111,6 +129,7 @@ type Generator struct {
 	depsolver         DepsolveFunc
 	containerResolver ContainerResolverFunc
 	commitResolver    CommitResolverFunc
+	sbomWriter        SBOMWriterFunc
 
 	reporegistry *reporegistry.RepoRegistry
 
@@ -131,6 +150,7 @@ func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, er
 		containerResolver: opts.ContainerResolver,
 		commitResolver:    opts.CommitResolver,
 		rpmDownloader:     opts.RpmDownloader,
+		sbomWriter:        opts.SBOMWriter,
 	}
 	if mg.out == nil {
 		mg.out = os.Stdout
@@ -190,5 +210,32 @@ func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgTy
 	}
 	fmt.Fprintf(mg.out, "%s\n", mf)
 
+	if mg.sbomWriter != nil {
+		// XXX: this is very similar to
+		// osbuild-composer:jobimpl-osbuild.go, see if code
+		// can be shared
+		for plName, depsolvedPipeline := range depsolved {
+			pipelinePurpose := "unknown"
+			switch {
+			case slices.Contains(imgType.PayloadPipelines(), plName):
+				pipelinePurpose = "image"
+			case slices.Contains(imgType.BuildPipelines(), plName):
+				pipelinePurpose = "buildroot"
+			}
+			// XXX: sync with image-builder-cli:build.go name generation - can we have a shared helper?
+			imageName := fmt.Sprintf("%s-%s-%s", dist.Name(), imgType.Name(), a.Name())
+			sbomDocOutputFilename := fmt.Sprintf("%s.%s-%s.spdx.json", imageName, pipelinePurpose, plName)
+
+			var buf bytes.Buffer
+			enc := json.NewEncoder(&buf)
+			if err := enc.Encode(depsolvedPipeline.SBOM); err != nil {
+				return err
+			}
+			if err := mg.sbomWriter(sbomDocOutputFilename, &buf); err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }

internal/manifestgen/manifestgen_test.go

@@ -3,7 +3,10 @@ package manifestgen_test
 import (
 	"bytes"
 	"crypto/sha256"
+	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"strings"
 	"testing"
 
@@ -20,6 +23,7 @@ import (
 	"github.com/osbuild/images/pkg/osbuild"
 	"github.com/osbuild/images/pkg/ostree"
 	"github.com/osbuild/images/pkg/rpmmd"
+	"github.com/osbuild/images/pkg/sbom"
 	testrepos "github.com/osbuild/images/test/data/repositories"
 
 	"github.com/osbuild/image-builder-cli/internal/manifestgen"
@@ -151,6 +155,11 @@ func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d
 					Id:       repoId,
 					Metalink: "https://example.com/metalink",
 				})
+				doc, err := sbom.NewDocument(sbom.StandardTypeSpdx, json.RawMessage(fmt.Sprintf(`{"sbom-for":"%s"}`, name)))
+				if err != nil {
+					return nil, err
+				}
+				resolvedSet.SBOM = doc
 			}
 		}
 		depsolvedSets[name] = resolvedSet
@@ -238,3 +247,45 @@ func TestManifestGeneratorContainers(t *testing.T) {
 	// container is included
 	assert.Contains(t, osbuildManifest.String(), "resolved-cnt-"+fakeContainerSource)
 }
+
+func TestManifestGeneratorDepsolveWithSbomWriter(t *testing.T) {
+	repos, err := testrepos.New()
+	assert.NoError(t, err)
+	fac := distrofactory.NewDefault()
+
+	filter, err := imagefilter.New(fac, repos)
+	assert.NoError(t, err)
+	res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64")
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(res))
+
+	var osbuildManifest bytes.Buffer
+	generatedSboms := map[string]string{}
+	opts := &manifestgen.Options{
+		Output:            &osbuildManifest,
+		Depsolver:         fakeDepsolve,
+		CommitResolver:    panicCommitResolver,
+		ContainerResolver: panicContainerResolver,
+
+		SBOMWriter: func(filename string, content io.Reader) error {
+			b, err := ioutil.ReadAll(content)
+			assert.NoError(t, err)
+			generatedSboms[filename] = strings.TrimSpace(string(b))
+			return nil
+		},
+	}
+	mg, err := manifestgen.New(repos, opts)
+	assert.NoError(t, err)
+	assert.NotNil(t, mg)
+	var bp blueprint.Blueprint
+	err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
+	require.NoError(t, err)
+
+	assert.Contains(t, generatedSboms, "centos-9-qcow2-x86_64.buildroot-build.spdx.json")
+	assert.Contains(t, generatedSboms, "centos-9-qcow2-x86_64.image-os.spdx.json")
+	expected := map[string]string{
+		"centos-9-qcow2-x86_64.buildroot-build.spdx.json": `{"DocType":"spdx","Document":{"sbom-for":"build"}}`,
+		"centos-9-qcow2-x86_64.image-os.spdx.json":        `{"DocType":"spdx","Document":{"sbom-for":"os"}}`,
+	}
+	assert.Equal(t, expected, generatedSboms)
+}

test/test_container.py

@@ -1,4 +1,6 @@
+import json
 import os
+import platform
 import subprocess
 
 import pytest
@@ -24,3 +26,29 @@ def test_container_builds_image(tmp_path, build_container, use_librepo):
     # XXX: ensure no other leftover dirs
     dents = os.listdir(output_dir)
     assert len(dents) == 1, f"too many dentries in output dir: {dents}"
+
+
+@pytest.mark.skipif(os.getuid() != 0, reason="needs root")
+def test_container_manifest_generates_sbom(tmp_path, build_container):
+    output_dir = tmp_path / "output"
+    output_dir.mkdir()
+    subprocess.check_call([
+        "podman", "run",
+        "--privileged",
+        "-v", f"{output_dir}:/output",
+        build_container,
+        "manifest",
+        "minimal-raw",
+        "--distro", "centos-9",
+        "--extra-artifacts=sbom",
+    ], stdout=subprocess.DEVNULL)
+    arch = platform.processor()
+    fn = f"centos-9-minimal-raw-{arch}/centos-9-minimal-raw-{arch}.image-os.spdx.json"
+    image_sbom_json_path = output_dir / fn
+    assert image_sbom_json_path.exists()
+    fn = f"centos-9-minimal-raw-{arch}/centos-9-minimal-raw-{arch}.buildroot-build.spdx.json"
+    buildroot_sbom_json_path = output_dir / fn
+    assert buildroot_sbom_json_path.exists()
+    sbom_json = json.loads(image_sbom_json_path.read_text())
+    # smoke test that we have glibc in the json doc
+    assert "glibc" in [s["name"] for s in sbom_json["Document"]["packages"]]