particle-os/debian-forge-cli
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ibcli: add new --extra-artifacts option with sbom support

Browse source 
This commit adds an option --extra-artifacts that can be
used to generate extra artifacts during the build or manifest
generation. Initially supported is `sbom` (but `manifest` is
planned too).

To use it run `--extra-artifacts=sbom` and it will generate
files like `centos-9-qcow2-x86_64.image-os.spdx.json` in
the output directory next to the generate runable artifact.

Closes: https://github.com/osbuild/image-builder-cli/issues/46
This commit is contained in:
Michael Vogt 2025-01-16 13:58:35 +01:00 • committed by Simon de Vlieger
parent db7cad2239
commit d485bc3a44
6 changed files with 178 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

7
cmd/image-builder/build.go
View file

@ -1,9 +1,6 @@
package main
import (
"fmt"
"path/filepath"
"github.com/osbuild/images/pkg/imagefilter"
"github.com/osbuild/images/pkg/osbuild"
)
@ -12,9 +9,7 @@ func buildImage(res *imagefilter.Result, osbuildManifest []byte, osbuildStoreDir
// XXX: support output dir via commandline
// XXX2: support output filename via commandline (c.f.
// https://github.com/osbuild/images/pull/1039)
outputDir := "."
buildName := fmt.Sprintf("%s-%s-%s", res.Distro.Name(), res.ImgType.Name(), res.Arch.Name())
jobOutputDir := filepath.Join(outputDir, buildName)
jobOutputDir := outputDirFor(res)
// XXX: support stremaing via images/pkg/osbuild/monitor.go
_, err := osbuild.RunOSBuild(osbuildManifest, osbuildStoreDir, jobOutputDir, res.ImgType.Exports(), nil, nil, false, osStderr)

17
cmd/image-builder/main.go
View file

@ -22,6 +22,11 @@ var (
osStderr io.Writer = os.Stderr
)
// generate the default output directory name for the given image
func outputDirFor(img *imagefilter.Result) string {
return fmt.Sprintf("%s-%s-%s", img.Distro.Name(), img.ImgType.Name(), img.Arch.Name())
}
func cmdListImages(cmd *cobra.Command, args []string) error {
filter, err := cmd.Flags().GetStringArray("filter")
if err != nil {
@ -80,6 +85,10 @@ func cmdManifestWrapper(cmd *cobra.Command, args []string, w io.Writer, archChec
if err != nil {
return nil, err
}
extraArtifacts, err := cmd.Flags().GetStringArray("extra-artifacts")
if err != nil {
return nil, err
}
ostreeImgOpts, err := ostreeImageOptions(cmd)
if err != nil {
return nil, err
@ -121,9 +130,10 @@ func cmdManifestWrapper(cmd *cobra.Command, args []string, w io.Writer, archChec
}
opts := &manifestOptions{
BlueprintPath: blueprintPath,
Ostree: ostreeImgOpts,
RpmDownloader: rpmDownloader,
BlueprintPath: blueprintPath,
Ostree: ostreeImgOpts,
RpmDownloader: rpmDownloader,
ExtraArtifacts: extraArtifacts,
}
err = generateManifest(dataDir, img, w, opts)
return img, err
@ -171,6 +181,7 @@ operating sytsems like centos and RHEL with easy customizations support.`,
SilenceErrors: true,
}
rootCmd.PersistentFlags().String("datadir", "", `Override the default data direcotry for e.g. custom repositories/*.json data`)
rootCmd.PersistentFlags().StringArray("extra-artifacts", nil, `Export extra artifacts to the output dir (e.g. "sbom")`)
rootCmd.SetOut(osStdout)
rootCmd.SetErr(osStderr)

39
cmd/image-builder/manifest.go
View file

@ -2,6 +2,9 @@ package main
import (
"io"
"os"
"path/filepath"
"slices"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/imagefilter"
@ -13,9 +16,23 @@ import (
)
type manifestOptions struct {
BlueprintPath string
Ostree *ostree.ImageOptions
RpmDownloader osbuild.RpmDownloader
BlueprintPath string
Ostree *ostree.ImageOptions
RpmDownloader osbuild.RpmDownloader
ExtraArtifacts []string
}
func sbomWriter(outputDir, filename string, content io.Reader) error {
p := filepath.Join(outputDir, filename)
f, err := os.Create(p)
if err != nil {
return err
}
defer f.Close()
if _, err := io.Copy(f, content); err != nil {
return err
}
return nil
}
func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer, opts *manifestOptions) error {
@ -24,13 +41,25 @@ func generateManifest(dataDir string, img *imagefilter.Result, output io.Writer,
return err
}
// XXX: add --rpmmd/cachedir option like bib
mg, err := manifestgen.New(repos, &manifestgen.Options{
manifestGenOpts := &manifestgen.Options{
Output: output,
RpmDownloader: opts.RpmDownloader,
})
}
if slices.Contains(opts.ExtraArtifacts, "sbom") {
outputDir := outputDirFor(img)
if err := os.MkdirAll(outputDir, 0755); err != nil {
return err
}
manifestGenOpts.SBOMWriter = func(filename string, content io.Reader) error {
return sbomWriter(outputDir, filename, content)
}
}
mg, err := manifestgen.New(repos, manifestGenOpts)
if err != nil {
return err
}
bp, err := blueprintload.Load(opts.BlueprintPath)
if err != nil {
return err

53
internal/manifestgen/manifestgen.go
View file

@ -1,9 +1,12 @@
package manifestgen
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"slices"
"strings"
"github.com/osbuild/images/pkg/blueprint"
@ -35,7 +38,9 @@ func defaultDepsolver(cacheDir string, packageSets map[string][]rpmmd.PackageSet
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir)
depsolvedSets := make(map[string]dnfjson.DepsolveResult)
for name, pkgSet := range packageSets {
res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone)
// XXX: is there harm in always generating an sbom?
// (expect for slightly longer runtime?)
res, err := solver.Depsolve(pkgSet, sbom.StandardTypeSpdx)
if err != nil {
return nil, fmt.Errorf("error depsolving: %w", err)
}
@ -88,18 +93,31 @@ type (
ContainerResolverFunc func(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error)
CommitResolverFunc func(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error)
SBOMWriterFunc func(filename string, content io.Reader) error
)
// Options contains the optional settings for the manifest generation.
// For unset values defaults will be used.
type Options struct {
Cachedir string
Output io.Writer
Cachedir string
Output io.Writer
// There are two types of sbom outputs, one for the "payload"
// and one for the "buildroot", we allow exporting both here
SbomImageOutput io.Writer
SbomBuildrootOutput io.Writer
Depsolver DepsolveFunc
ContainerResolver ContainerResolverFunc
CommitResolver CommitResolverFunc
RpmDownloader osbuild.RpmDownloader
// Will be called for each generated SBOM the filename
// contains the suggest filename string and the content
// can be read
SBOMWriter SBOMWriterFunc
}
// Generator can generate an osbuild manifest from a given repository
@ -111,6 +129,7 @@ type Generator struct {
depsolver DepsolveFunc
containerResolver ContainerResolverFunc
commitResolver CommitResolverFunc
sbomWriter SBOMWriterFunc
reporegistry *reporegistry.RepoRegistry
@ -131,6 +150,7 @@ func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, er
containerResolver: opts.ContainerResolver,
commitResolver: opts.CommitResolver,
rpmDownloader: opts.RpmDownloader,
sbomWriter: opts.SBOMWriter,
}
if mg.out == nil {
mg.out = os.Stdout
@ -190,5 +210,32 @@ func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgTy
}
fmt.Fprintf(mg.out, "%s\n", mf)
if mg.sbomWriter != nil {
// XXX: this is very similar to
// osbuild-composer:jobimpl-osbuild.go, see if code
// can be shared
for plName, depsolvedPipeline := range depsolved {
pipelinePurpose := "unknown"
switch {
case slices.Contains(imgType.PayloadPipelines(), plName):
pipelinePurpose = "image"
case slices.Contains(imgType.BuildPipelines(), plName):
pipelinePurpose = "buildroot"
}
// XXX: sync with image-builder-cli:build.go name generation - can we have a shared helper?
imageName := fmt.Sprintf("%s-%s-%s", dist.Name(), imgType.Name(), a.Name())
sbomDocOutputFilename := fmt.Sprintf("%s.%s-%s.spdx.json", imageName, pipelinePurpose, plName)
var buf bytes.Buffer
enc := json.NewEncoder(&buf)
if err := enc.Encode(depsolvedPipeline.SBOM); err != nil {
return err
}
if err := mg.sbomWriter(sbomDocOutputFilename, &buf); err != nil {
return err
}
}
}
return nil
}

51
internal/manifestgen/manifestgen_test.go
View file

@ -3,7 +3,10 @@ package manifestgen_test
import (
"bytes"
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"strings"
"testing"
@ -20,6 +23,7 @@ import (
"github.com/osbuild/images/pkg/osbuild"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/images/pkg/sbom"
testrepos "github.com/osbuild/images/test/data/repositories"
"github.com/osbuild/image-builder-cli/internal/manifestgen"
@ -151,6 +155,11 @@ func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d
Id: repoId,
Metalink: "https://example.com/metalink",
})
doc, err := sbom.NewDocument(sbom.StandardTypeSpdx, json.RawMessage(fmt.Sprintf(`{"sbom-for":"%s"}`, name)))
if err != nil {
return nil, err
}
resolvedSet.SBOM = doc
}
}
depsolvedSets[name] = resolvedSet
@ -238,3 +247,45 @@ func TestManifestGeneratorContainers(t *testing.T) {
// container is included
assert.Contains(t, osbuildManifest.String(), "resolved-cnt-"+fakeContainerSource)
}
func TestManifestGeneratorDepsolveWithSbomWriter(t *testing.T) {
repos, err := testrepos.New()
assert.NoError(t, err)
fac := distrofactory.NewDefault()
filter, err := imagefilter.New(fac, repos)
assert.NoError(t, err)
res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64")
assert.NoError(t, err)
assert.Equal(t, 1, len(res))
var osbuildManifest bytes.Buffer
generatedSboms := map[string]string{}
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
CommitResolver: panicCommitResolver,
ContainerResolver: panicContainerResolver,
SBOMWriter: func(filename string, content io.Reader) error {
b, err := ioutil.ReadAll(content)
assert.NoError(t, err)
generatedSboms[filename] = strings.TrimSpace(string(b))
return nil
},
}
mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)
assert.NotNil(t, mg)
var bp blueprint.Blueprint
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
require.NoError(t, err)
assert.Contains(t, generatedSboms, "centos-9-qcow2-x86_64.buildroot-build.spdx.json")
assert.Contains(t, generatedSboms, "centos-9-qcow2-x86_64.image-os.spdx.json")
expected := map[string]string{
"centos-9-qcow2-x86_64.buildroot-build.spdx.json": `{"DocType":"spdx","Document":{"sbom-for":"build"}}`,
"centos-9-qcow2-x86_64.image-os.spdx.json": `{"DocType":"spdx","Document":{"sbom-for":"os"}}`,
}
assert.Equal(t, expected, generatedSboms)
}

28
test/test_container.py
View file

@ -1,4 +1,6 @@
import json
import os
import platform
import subprocess
import pytest
@ -24,3 +26,29 @@ def test_container_builds_image(tmp_path, build_container, use_librepo):
# XXX: ensure no other leftover dirs
dents = os.listdir(output_dir)
assert len(dents) == 1, f"too many dentries in output dir: {dents}"
@pytest.mark.skipif(os.getuid() != 0, reason="needs root")
def test_container_manifest_generates_sbom(tmp_path, build_container):
output_dir = tmp_path / "output"
output_dir.mkdir()
subprocess.check_call([
"podman", "run",
"--privileged",
"-v", f"{output_dir}:/output",
build_container,
"manifest",
"minimal-raw",
"--distro", "centos-9",
"--extra-artifacts=sbom",
], stdout=subprocess.DEVNULL)
arch = platform.processor()
fn = f"centos-9-minimal-raw-{arch}/centos-9-minimal-raw-{arch}.image-os.spdx.json"
image_sbom_json_path = output_dir / fn
assert image_sbom_json_path.exists()
fn = f"centos-9-minimal-raw-{arch}/centos-9-minimal-raw-{arch}.buildroot-build.spdx.json"
buildroot_sbom_json_path = output_dir / fn
assert buildroot_sbom_json_path.exists()
sbom_json = json.loads(image_sbom_json_path.read_text())
# smoke test that we have glibc in the json doc
assert "glibc" in [s["name"] for s in sbom_json["Document"]["packages"]]