From ea61ef593fb6c2f6ed65f7582bbaa7f17bb5ef9e Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Mon, 2 Dec 2024 12:17:53 +0100 Subject: [PATCH] pkg: add new `manifestgen` package This commit adds a new generic `manifestgen` package that can be used to generate osbuild manifests. It works on a higher level then the low-level `manifest` package from `images` and provides plugable resolvers and a streamlined API. This package is meant to be moved to the `images` library eventually. --- go.mod | 2 +- internal/manifestgen/manifestgen.go | 184 +++++++++++++++++++ internal/manifestgen/manifestgen_test.go | 217 +++++++++++++++++++++++ 3 files changed, 402 insertions(+), 1 deletion(-) create mode 100644 internal/manifestgen/manifestgen.go create mode 100644 internal/manifestgen/manifestgen_test.go diff --git a/go.mod b/go.mod index 3e44268..aa5c4a3 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ module github.com/osbuild/image-builder-cli go 1.21.0 require ( + github.com/gobwas/glob v0.2.3 github.com/osbuild/images v0.98.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 @@ -50,7 +51,6 @@ require ( github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-openapi/validate v0.24.0 // indirect - github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/internal/manifestgen/manifestgen.go b/internal/manifestgen/manifestgen.go new file mode 100644 index 0000000..1c912eb --- /dev/null +++ b/internal/manifestgen/manifestgen.go @@ -0,0 +1,184 @@ +package manifestgen + +import ( + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/osbuild/images/pkg/blueprint" + "github.com/osbuild/images/pkg/container" + "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/dnfjson" + "github.com/osbuild/images/pkg/ostree" + "github.com/osbuild/images/pkg/reporegistry" + "github.com/osbuild/images/pkg/rpmmd" + "github.com/osbuild/images/pkg/sbom" +) + +// XXX: all of the helpers below are duplicated from +// cmd/build/main.go:depsolve (and probably more places) should go +// into a common helper in "images" or images should do this on its +// own +func defaultDepsolver(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { + solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) + depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) + for name, pkgSet := range packageSets { + res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone) + if err != nil { + return nil, nil, fmt.Errorf("error depsolving: %w", err) + } + depsolvedSets[name] = res.Packages + repoSets[name] = res.Repos + // the depsolve result also contains SBOM information, + // it is currently not used here though + } + return depsolvedSets, repoSets, nil +} + +func resolveContainers(containers []container.SourceSpec, archName string) ([]container.Spec, error) { + resolver := container.NewResolver(archName) + + for _, c := range containers { + resolver.Add(c) + } + + return resolver.Finish() +} + +func defaultContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { + containerSpecs := make(map[string][]container.Spec, len(containerSources)) + for plName, sourceSpecs := range containerSources { + specs, err := resolveContainers(sourceSpecs, archName) + if err != nil { + return nil, fmt.Errorf("error container resolving: %w", err) + } + containerSpecs[plName] = specs + } + return containerSpecs, nil +} + +func defaultCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { + commits := make(map[string][]ostree.CommitSpec, len(commitSources)) + for name, commitSources := range commitSources { + commitSpecs := make([]ostree.CommitSpec, len(commitSources)) + for idx, commitSource := range commitSources { + var err error + commitSpecs[idx], err = ostree.Resolve(commitSource) + if err != nil { + return nil, fmt.Errorf("error ostree commit resolving: %w", err) + } + } + commits[name] = commitSpecs + } + return commits, nil +} + +type ( + DepsolveFunc func(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) + + ContainerResolverFunc func(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) + + CommitResolverFunc func(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) +) + +// Options contains the optional settings for the manifest generation. +// For unset values defaults will be used. +type Options struct { + Cachedir string + Output io.Writer + Depsolver DepsolveFunc + ContainerResolver ContainerResolverFunc + CommitResolver CommitResolverFunc +} + +// Generator can generate an osbuild manifest from a given repository +// and options. +type Generator struct { + cacheDir string + out io.Writer + + depsolver DepsolveFunc + containerResolver ContainerResolverFunc + commitResolver CommitResolverFunc + + reporegistry *reporegistry.RepoRegistry +} + +// New will create a new manifest generator +func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, error) { + if opts == nil { + opts = &Options{} + } + mg := &Generator{ + reporegistry: reporegistry, + + cacheDir: opts.Cachedir, + out: opts.Output, + depsolver: opts.Depsolver, + containerResolver: opts.ContainerResolver, + commitResolver: opts.CommitResolver, + } + if mg.out == nil { + mg.out = os.Stdout + } + if mg.depsolver == nil { + mg.depsolver = defaultDepsolver + } + if mg.containerResolver == nil { + mg.containerResolver = defaultContainerResolver + } + if mg.commitResolver == nil { + mg.commitResolver = defaultCommitResolver + } + + return mg, nil +} + +// Generate will generate a new manifest for the given distro/imageType/arch +// combination. +func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgType distro.ImageType, a distro.Arch, imgOpts *distro.ImageOptions) error { + if imgOpts == nil { + imgOpts = &distro.ImageOptions{} + } + // we may allow to customize the seed in the future via imgOpts or + // an environment variable + // XXX: look into "images" so that it automatically seeds when pasing + // a "0" seed. + seed := time.Now().UnixNano() + + repos, err := mg.reporegistry.ReposByImageTypeName(dist.Name(), a.Name(), imgType.Name()) + if err != nil { + return err + } + preManifest, warnings, err := imgType.Manifest(bp, *imgOpts, repos, seed) + if err != nil { + return err + } + if len(warnings) > 0 { + // XXX: what can we do here? for things like json output? + // what are these warnings? + return fmt.Errorf("warnings during manifest creation: %v", strings.Join(warnings, "\n")) + } + packageSpecs, _, err := mg.depsolver(mg.cacheDir, preManifest.GetPackageSetChains(), dist, a.Name()) + if err != nil { + return err + } + containerSpecs, err := mg.containerResolver(preManifest.GetContainerSourceSpecs(), a.Name()) + if err != nil { + return err + } + commitSpecs, err := mg.commitResolver(preManifest.GetOSTreeSourceSpecs()) + if err != nil { + return err + } + mf, err := preManifest.Serialize(packageSpecs, containerSpecs, commitSpecs, nil) + if err != nil { + return err + } + fmt.Fprintf(mg.out, "%s\n", mf) + + return nil +} diff --git a/internal/manifestgen/manifestgen_test.go b/internal/manifestgen/manifestgen_test.go new file mode 100644 index 0000000..91128cb --- /dev/null +++ b/internal/manifestgen/manifestgen_test.go @@ -0,0 +1,217 @@ +package manifestgen_test + +import ( + "bytes" + "crypto/sha256" + "fmt" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + + "github.com/osbuild/images/pkg/blueprint" + "github.com/osbuild/images/pkg/container" + "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/distrofactory" + "github.com/osbuild/images/pkg/imagefilter" + "github.com/osbuild/images/pkg/ostree" + "github.com/osbuild/images/pkg/rpmmd" + testrepos "github.com/osbuild/images/test/data/repositories" + + "github.com/osbuild/image-builder-cli/internal/manifestgen" + "github.com/osbuild/image-builder-cli/internal/manifesttest" +) + +func init() { + // silence logrus by default, it is quite verbose + logrus.SetLevel(logrus.WarnLevel) +} + +func sha256For(s string) string { + h := sha256.New() + h.Write([]byte(s)) + bs := h.Sum(nil) + return fmt.Sprintf("sha256:%x", bs) +} + +func TestManifestGeneratorDepsolve(t *testing.T) { + repos, err := testrepos.New() + assert.NoError(t, err) + fac := distrofactory.NewDefault() + + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + var osbuildManifest bytes.Buffer + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: panicCommitResolver, + ContainerResolver: panicContainerResolver, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + var bp blueprint.Blueprint + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) + assert.NoError(t, err) + + pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes()) + assert.NoError(t, err) + assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames) + + // we expect at least a "kernel" package in the manifest, + // sadly the test distro does not really generate much here so we + // need to use this as a canary that resolving happend + // XXX: add testhelper to manifesttest for this + expectedSha256 := sha256For("kernel") + assert.Contains(t, osbuildManifest.String(), expectedSha256) +} + +func TestManifestGeneratorWithOstreeCommit(t *testing.T) { + var osbuildManifest bytes.Buffer + + repos, err := testrepos.New() + assert.NoError(t, err) + + fac := distrofactory.NewDefault() + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:edge-ami", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: fakeCommitResolver, + ContainerResolver: panicContainerResolver, + } + imageOpts := &distro.ImageOptions{ + OSTree: &ostree.ImageOptions{ + //ImageRef: "latest/1/x86_64/edge", + URL: "http://example.com/", + }, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + var bp blueprint.Blueprint + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, imageOpts) + assert.NoError(t, err) + + pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes()) + assert.NoError(t, err) + assert.Equal(t, []string{"build", "ostree-deployment", "image"}, pipelineNames) + + // XXX: add testhelper to manifesttest for this + assert.Contains(t, osbuildManifest.String(), `{"url":"resolved-url-for-centos/9/x86_64/edge"}`) + // we expect at least a "glibc" package in the manifest, + // sadly the test distro does not really generate much here so we + // need to use this as a canary that resolving happend + // XXX: add testhelper to manifesttest for this + expectedSha256 := sha256For("glibc") + assert.Contains(t, osbuildManifest.String(), expectedSha256) +} + +func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) { + depsolvedSets := make(map[string][]rpmmd.PackageSpec) + repoSets := make(map[string][]rpmmd.RepoConfig) + for name, pkgSets := range packageSets { + var resolvedSet []rpmmd.PackageSpec + for _, pkgSet := range pkgSets { + for _, pkgName := range pkgSet.Include { + resolvedSet = append(resolvedSet, rpmmd.PackageSpec{ + Name: pkgName, + Checksum: sha256For(pkgName), + }) + } + } + depsolvedSets[name] = resolvedSet + } + return depsolvedSets, repoSets, nil +} + +func fakeCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { + commits := make(map[string][]ostree.CommitSpec, len(commitSources)) + for name, commitSources := range commitSources { + commitSpecs := make([]ostree.CommitSpec, len(commitSources)) + for idx, commitSource := range commitSources { + commitSpecs[idx] = ostree.CommitSpec{ + URL: fmt.Sprintf("resolved-url-for-%s", commitSource.Ref), + } + } + commits[name] = commitSpecs + } + return commits, nil + +} + +func panicCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { + if len(commitSources) > 0 { + panic("panicCommitResolver") + } + return nil, nil +} + +func fakeContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { + containerSpecs := make(map[string][]container.Spec, len(containerSources)) + for plName, sourceSpecs := range containerSources { + var containers []container.Spec + for _, spec := range sourceSpecs { + containers = append(containers, container.Spec{ + Source: fmt.Sprintf("resolved-cnt-%s", spec.Source), + Digest: "sha256:" + sha256For("digest:"+spec.Source), + ImageID: "sha256:" + sha256For("id:"+spec.Source), + }) + } + containerSpecs[plName] = containers + } + return containerSpecs, nil +} + +func panicContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { + if len(containerSources) > 0 { + panic("panicContainerResolver") + } + return nil, nil +} + +func TestManifestGeneratorContainers(t *testing.T) { + repos, err := testrepos.New() + assert.NoError(t, err) + fac := distrofactory.NewDefault() + + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + var osbuildManifest bytes.Buffer + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: panicCommitResolver, + ContainerResolver: fakeContainerResolver, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + fakeContainerSource := "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal" + bp := blueprint.Blueprint{ + Containers: []blueprint.Container{ + { + Source: fakeContainerSource, + }, + }, + } + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) + assert.NoError(t, err) + + // container is included + assert.Contains(t, osbuildManifest.String(), "resolved-cnt-"+fakeContainerSource) +}