particle-os/debian-forge-cli
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

pkg: add new manifestgen package

Browse source 
This commit adds a new generic `manifestgen` package that can be
used to generate osbuild manifests. It works on a higher level
then the low-level `manifest` package from `images` and provides
plugable resolvers and a streamlined API.

This package is meant to be moved to the `images` library eventually.
This commit is contained in:
Michael Vogt 2024-12-02 12:17:53 +01:00 • committed by Simon de Vlieger
parent 5a6ee5c1ca
commit ea61ef593f
3 changed files with 402 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -5,6 +5,7 @@ module github.com/osbuild/image-builder-cli
go 1.21.0
require (
github.com/gobwas/glob v0.2.3
github.com/osbuild/images v0.98.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.1
@ -50,7 +51,6 @@ require (
github.com/go-openapi/strfmt v0.23.0 // indirect
github.com/go-openapi/swag v0.23.0 // indirect
github.com/go-openapi/validate v0.24.0 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect

184
internal/manifestgen/manifestgen.go Normal file
View file

@ -0,0 +1,184 @@
package manifestgen
import (
"fmt"
"io"
"os"
"strings"
"time"
"github.com/osbuild/images/pkg/blueprint"
"github.com/osbuild/images/pkg/container"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/dnfjson"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/reporegistry"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/images/pkg/sbom"
)
// XXX: all of the helpers below are duplicated from
// cmd/build/main.go:depsolve (and probably more places) should go
// into a common helper in "images" or images should do this on its
// own
func defaultDepsolver(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) {
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir)
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
repoSets := make(map[string][]rpmmd.RepoConfig)
for name, pkgSet := range packageSets {
res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone)
if err != nil {
return nil, nil, fmt.Errorf("error depsolving: %w", err)
}
depsolvedSets[name] = res.Packages
repoSets[name] = res.Repos
// the depsolve result also contains SBOM information,
// it is currently not used here though
}
return depsolvedSets, repoSets, nil
}
func resolveContainers(containers []container.SourceSpec, archName string) ([]container.Spec, error) {
resolver := container.NewResolver(archName)
for _, c := range containers {
resolver.Add(c)
}
return resolver.Finish()
}
func defaultContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) {
containerSpecs := make(map[string][]container.Spec, len(containerSources))
for plName, sourceSpecs := range containerSources {
specs, err := resolveContainers(sourceSpecs, archName)
if err != nil {
return nil, fmt.Errorf("error container resolving: %w", err)
}
containerSpecs[plName] = specs
}
return containerSpecs, nil
}
func defaultCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) {
commits := make(map[string][]ostree.CommitSpec, len(commitSources))
for name, commitSources := range commitSources {
commitSpecs := make([]ostree.CommitSpec, len(commitSources))
for idx, commitSource := range commitSources {
var err error
commitSpecs[idx], err = ostree.Resolve(commitSource)
if err != nil {
return nil, fmt.Errorf("error ostree commit resolving: %w", err)
}
}
commits[name] = commitSpecs
}
return commits, nil
}
type (
DepsolveFunc func(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error)
ContainerResolverFunc func(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error)
CommitResolverFunc func(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error)
)
// Options contains the optional settings for the manifest generation.
// For unset values defaults will be used.
type Options struct {
Cachedir string
Output io.Writer
Depsolver DepsolveFunc
ContainerResolver ContainerResolverFunc
CommitResolver CommitResolverFunc
}
// Generator can generate an osbuild manifest from a given repository
// and options.
type Generator struct {
cacheDir string
out io.Writer
depsolver DepsolveFunc
containerResolver ContainerResolverFunc
commitResolver CommitResolverFunc
reporegistry *reporegistry.RepoRegistry
}
// New will create a new manifest generator
func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, error) {
if opts == nil {
opts = &Options{}
}
mg := &Generator{
reporegistry: reporegistry,
cacheDir: opts.Cachedir,
out: opts.Output,
depsolver: opts.Depsolver,
containerResolver: opts.ContainerResolver,
commitResolver: opts.CommitResolver,
}
if mg.out == nil {
mg.out = os.Stdout
}
if mg.depsolver == nil {
mg.depsolver = defaultDepsolver
}
if mg.containerResolver == nil {
mg.containerResolver = defaultContainerResolver
}
if mg.commitResolver == nil {
mg.commitResolver = defaultCommitResolver
}
return mg, nil
}
// Generate will generate a new manifest for the given distro/imageType/arch
// combination.
func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgType distro.ImageType, a distro.Arch, imgOpts *distro.ImageOptions) error {
if imgOpts == nil {
imgOpts = &distro.ImageOptions{}
}
// we may allow to customize the seed in the future via imgOpts or
// an environment variable
// XXX: look into "images" so that it automatically seeds when pasing
// a "0" seed.
seed := time.Now().UnixNano()
repos, err := mg.reporegistry.ReposByImageTypeName(dist.Name(), a.Name(), imgType.Name())
if err != nil {
return err
}
preManifest, warnings, err := imgType.Manifest(bp, *imgOpts, repos, seed)
if err != nil {
return err
}
if len(warnings) > 0 {
// XXX: what can we do here? for things like json output?
// what are these warnings?
return fmt.Errorf("warnings during manifest creation: %v", strings.Join(warnings, "\n"))
}
packageSpecs, _, err := mg.depsolver(mg.cacheDir, preManifest.GetPackageSetChains(), dist, a.Name())
if err != nil {
return err
}
containerSpecs, err := mg.containerResolver(preManifest.GetContainerSourceSpecs(), a.Name())
if err != nil {
return err
}
commitSpecs, err := mg.commitResolver(preManifest.GetOSTreeSourceSpecs())
if err != nil {
return err
}
mf, err := preManifest.Serialize(packageSpecs, containerSpecs, commitSpecs, nil)
if err != nil {
return err
}
fmt.Fprintf(mg.out, "%s\n", mf)
return nil
}

217
internal/manifestgen/manifestgen_test.go Normal file
View file

@ -0,0 +1,217 @@
package manifestgen_test
import (
"bytes"
"crypto/sha256"
"fmt"
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/osbuild/images/pkg/blueprint"
"github.com/osbuild/images/pkg/container"
"github.com/osbuild/images/pkg/distro"
"github.com/osbuild/images/pkg/distrofactory"
"github.com/osbuild/images/pkg/imagefilter"
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/rpmmd"
testrepos "github.com/osbuild/images/test/data/repositories"
"github.com/osbuild/image-builder-cli/internal/manifestgen"
"github.com/osbuild/image-builder-cli/internal/manifesttest"
)
func init() {
// silence logrus by default, it is quite verbose
logrus.SetLevel(logrus.WarnLevel)
}
func sha256For(s string) string {
h := sha256.New()
h.Write([]byte(s))
bs := h.Sum(nil)
return fmt.Sprintf("sha256:%x", bs)
}
func TestManifestGeneratorDepsolve(t *testing.T) {
repos, err := testrepos.New()
assert.NoError(t, err)
fac := distrofactory.NewDefault()
filter, err := imagefilter.New(fac, repos)
assert.NoError(t, err)
res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64")
assert.NoError(t, err)
assert.Equal(t, 1, len(res))
var osbuildManifest bytes.Buffer
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
CommitResolver: panicCommitResolver,
ContainerResolver: panicContainerResolver,
}
mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)
assert.NotNil(t, mg)
var bp blueprint.Blueprint
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
assert.NoError(t, err)
pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes())
assert.NoError(t, err)
assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames)
// we expect at least a "kernel" package in the manifest,
// sadly the test distro does not really generate much here so we
// need to use this as a canary that resolving happend
// XXX: add testhelper to manifesttest for this
expectedSha256 := sha256For("kernel")
assert.Contains(t, osbuildManifest.String(), expectedSha256)
}
func TestManifestGeneratorWithOstreeCommit(t *testing.T) {
var osbuildManifest bytes.Buffer
repos, err := testrepos.New()
assert.NoError(t, err)
fac := distrofactory.NewDefault()
filter, err := imagefilter.New(fac, repos)
assert.NoError(t, err)
res, err := filter.Filter("distro:centos-9", "type:edge-ami", "arch:x86_64")
assert.NoError(t, err)
assert.Equal(t, 1, len(res))
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
CommitResolver: fakeCommitResolver,
ContainerResolver: panicContainerResolver,
}
imageOpts := &distro.ImageOptions{
OSTree: &ostree.ImageOptions{
//ImageRef: "latest/1/x86_64/edge",
URL: "http://example.com/",
},
}
mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)
assert.NotNil(t, mg)
var bp blueprint.Blueprint
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, imageOpts)
assert.NoError(t, err)
pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes())
assert.NoError(t, err)
assert.Equal(t, []string{"build", "ostree-deployment", "image"}, pipelineNames)
// XXX: add testhelper to manifesttest for this
assert.Contains(t, osbuildManifest.String(), `{"url":"resolved-url-for-centos/9/x86_64/edge"}`)
// we expect at least a "glibc" package in the manifest,
// sadly the test distro does not really generate much here so we
// need to use this as a canary that resolving happend
// XXX: add testhelper to manifesttest for this
expectedSha256 := sha256For("glibc")
assert.Contains(t, osbuildManifest.String(), expectedSha256)
}
func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, map[string][]rpmmd.RepoConfig, error) {
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
repoSets := make(map[string][]rpmmd.RepoConfig)
for name, pkgSets := range packageSets {
var resolvedSet []rpmmd.PackageSpec
for _, pkgSet := range pkgSets {
for _, pkgName := range pkgSet.Include {
resolvedSet = append(resolvedSet, rpmmd.PackageSpec{
Name: pkgName,
Checksum: sha256For(pkgName),
})
}
}
depsolvedSets[name] = resolvedSet
}
return depsolvedSets, repoSets, nil
}
func fakeCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) {
commits := make(map[string][]ostree.CommitSpec, len(commitSources))
for name, commitSources := range commitSources {
commitSpecs := make([]ostree.CommitSpec, len(commitSources))
for idx, commitSource := range commitSources {
commitSpecs[idx] = ostree.CommitSpec{
URL: fmt.Sprintf("resolved-url-for-%s", commitSource.Ref),
}
}
commits[name] = commitSpecs
}
return commits, nil
}
func panicCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) {
if len(commitSources) > 0 {
panic("panicCommitResolver")
}
return nil, nil
}
func fakeContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) {
containerSpecs := make(map[string][]container.Spec, len(containerSources))
for plName, sourceSpecs := range containerSources {
var containers []container.Spec
for _, spec := range sourceSpecs {
containers = append(containers, container.Spec{
Source: fmt.Sprintf("resolved-cnt-%s", spec.Source),
Digest: "sha256:" + sha256For("digest:"+spec.Source),
ImageID: "sha256:" + sha256For("id:"+spec.Source),
})
}
containerSpecs[plName] = containers
}
return containerSpecs, nil
}
func panicContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) {
if len(containerSources) > 0 {
panic("panicContainerResolver")
}
return nil, nil
}
func TestManifestGeneratorContainers(t *testing.T) {
repos, err := testrepos.New()
assert.NoError(t, err)
fac := distrofactory.NewDefault()
filter, err := imagefilter.New(fac, repos)
assert.NoError(t, err)
res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64")
assert.NoError(t, err)
assert.Equal(t, 1, len(res))
var osbuildManifest bytes.Buffer
opts := &manifestgen.Options{
Output: &osbuildManifest,
Depsolver: fakeDepsolve,
CommitResolver: panicCommitResolver,
ContainerResolver: fakeContainerResolver,
}
mg, err := manifestgen.New(repos, opts)
assert.NoError(t, err)
assert.NotNil(t, mg)
fakeContainerSource := "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal"
bp := blueprint.Blueprint{
Containers: []blueprint.Container{
{
Source: fakeContainerSource,
},
},
}
err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil)
assert.NoError(t, err)
// container is included
assert.Contains(t, osbuildManifest.String(), "resolved-cnt-"+fakeContainerSource)
}