particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

manifest: remove Content field from manifest

Browse source 
Do not expose the content of the manifest statically and instead rely on
the public methods to retrieve source specifications dynamically.

Since the methods require iterating through the pipelines to collect
source specifications, we should avoid calling the function multiple
times when we can reuse the returned values.
This commit is contained in:
Achilleas Koutsou 2023-06-02 17:39:29 +02:00 • committed by Ondřej Budai
parent cecbc97e06
commit 5dba246813
20 changed files with 45 additions and 82 deletions
Download patch file Download diff file Expand all files Collapse all files

6
cmd/gen-manifests/main.go
View file

@ -158,7 +158,7 @@ func makeManifestJob(name string, imgType distro.ImageType, cr composeRequest, d
return
}
packageSpecs, err := depsolve(cacheDir, manifest.Content.PackageSets, distribution, archName)
packageSpecs, err := depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName)
if err != nil {
err = fmt.Errorf("[%s] depsolve failed: %s", filename, err.Error())
return
@ -172,12 +172,12 @@ func makeManifestJob(name string, imgType distro.ImageType, cr composeRequest, d
bp = blueprint.Blueprint(*cr.Blueprint)
}
containerSpecs, err := resolvePipelineContainers(manifest.Content.Containers, archName)
containerSpecs, err := resolvePipelineContainers(manifest.GetContainerSourceSpecs(), archName)
if err != nil {
return fmt.Errorf("[%s] container resolution failed: %s", filename, err.Error())
}
commitSpecs := resolvePipelineCommits(manifest.Content.OSTreeCommits)
commitSpecs := resolvePipelineCommits(manifest.GetOSTreeSourceSpecs())
mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs)
if err != nil {

4
cmd/osbuild-dnf-json-tests/main_test.go
View file

@ -64,7 +64,7 @@ func TestCrossArchDepsolve(t *testing.T) {
repos[archStr], 0)
assert.NoError(t, err)
for _, set := range manifest.Content.PackageSets {
for _, set := range manifest.GetPackageSetChains() {
_, err = solver.Depsolve(set)
assert.NoError(t, err)
}
@ -101,7 +101,7 @@ func TestDepsolvePackageSets(t *testing.T) {
manifestSource, _, err := qcow2Image.Manifest(&blueprint.Blueprint{Packages: []blueprint.Package{{Name: "bind"}}}, distro.ImageOptions{}, x86Repos, 0)
require.Nilf(t, err, "failed to initialise manifest for %q image type of %q/%q distro/arch", qcow2ImageTypeName, cs9.Name(), platform.ARCH_X86_64.String())
imagePkgSets := manifestSource.Content.PackageSets
imagePkgSets := manifestSource.GetPackageSetChains()
gotPackageSpecsSets := make(map[string][]rpmmd.PackageSpec, len(imagePkgSets))
for name, pkgSet := range imagePkgSets {

2
cmd/osbuild-package-sets/main.go
View file

@ -57,5 +57,5 @@ func main() {
if err != nil {
panic(err)
}
_ = encoder.Encode(manifest.Content.PackageSets)
_ = encoder.Encode(manifest.GetPackageSetChains())
}

12
cmd/osbuild-pipeline/main.go
View file

@ -201,7 +201,7 @@ func main() {
}
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
for name, pkgSet := range manifest.Content.PackageSets {
for name, pkgSet := range manifest.GetPackageSetChains() {
res, err := solver.Depsolve(pkgSet)
if err != nil {
panic("Could not depsolve: " + err.Error())
@ -209,8 +209,9 @@ func main() {
depsolvedSets[name] = res
}
containers := make(map[string][]container.Spec, len(manifest.Content.Containers))
for name, sourceSpecs := range manifest.Content.Containers {
containerSources := manifest.GetContainerSourceSpecs()
containers := make(map[string][]container.Spec, len(containerSources))
for name, sourceSpecs := range containerSources {
containerSpecs, err := resolveContainers(sourceSpecs, arch.Name())
if err != nil {
panic("Could not resolve containers: " + err.Error())
@ -219,8 +220,9 @@ func main() {
}
// "resolve" ostree commits by copying the source specs into commit specs
commits := make(map[string][]ostree.CommitSpec, len(manifest.Content.OSTreeCommits))
for name, commitSources := range manifest.Content.OSTreeCommits {
commitSources := manifest.GetOSTreeSourceSpecs()
commits := make(map[string][]ostree.CommitSpec, len(commitSources))
for name, commitSources := range commitSources {
commitSpecs := make([]ostree.CommitSpec, len(commitSources))
for idx, commitSource := range commitSources {
commitSpecs[idx] = ostree.CommitSpec{

2
cmd/osbuild-store-dump/main.go
View file

@ -28,7 +28,7 @@ func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d di
}
pkgSpecSets := make(map[string][]rpmmd.PackageSpec)
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), a.Name(), d.Name(), cacheDir)
for name, packages := range manifest.Content.PackageSets {
for name, packages := range manifest.GetPackageSetChains() {
res, err := solver.Depsolve(packages)
if err != nil {
panic(err)

16
internal/cloudapi/v2/server.go
View file

@ -114,7 +114,7 @@ func (s *Server) enqueueCompose(distribution distro.Distro, bp blueprint.Bluepri
}
depsolveJobID, err := s.workers.EnqueueDepsolve(&worker.DepsolveJob{
PackageSets: manifestSource.Content.PackageSets,
PackageSets: manifestSource.GetPackageSetChains(),
ModulePlatformID: distribution.ModulePlatformID(),
Arch: ir.arch.Name(),
Releasever: distribution.Releasever(),
@ -125,7 +125,7 @@ func (s *Server) enqueueCompose(distribution distro.Distro, bp blueprint.Bluepri
dependencies := []uuid.UUID{depsolveJobID}
var containerResolveJobID uuid.UUID
containerSources := manifestSource.Content.Containers
containerSources := manifestSource.GetContainerSourceSpecs()
if len(containerSources) > 1 {
// only one pipeline can embed containers
pipelines := make([]string, 0, len(containerSources))
@ -161,7 +161,7 @@ func (s *Server) enqueueCompose(distribution distro.Distro, bp blueprint.Bluepri
}
var ostreeResolveJobID uuid.UUID
commitSources := manifestSource.Content.OSTreeCommits
commitSources := manifestSource.GetOSTreeSourceSpecs()
if len(commitSources) > 1 {
// only one pipeline can specify an ostree commit for content
pipelines := make([]string, 0, len(commitSources))
@ -234,7 +234,7 @@ func (s *Server) enqueueKojiCompose(taskID uint64, server, name, version, releas
}
depsolveJobID, err := s.workers.EnqueueDepsolve(&worker.DepsolveJob{
PackageSets: manifestSource.Content.PackageSets,
PackageSets: manifestSource.GetPackageSetChains(),
ModulePlatformID: distribution.ModulePlatformID(),
Arch: ir.arch.Name(),
Releasever: distribution.Releasever(),
@ -245,7 +245,7 @@ func (s *Server) enqueueKojiCompose(taskID uint64, server, name, version, releas
dependencies := []uuid.UUID{depsolveJobID}
var containerResolveJobID uuid.UUID
containerSources := manifestSource.Content.Containers
containerSources := manifestSource.GetContainerSourceSpecs()
if len(containerSources) > 1 {
// only one pipeline can embed containers
pipelines := make([]string, 0, len(containerSources))
@ -281,7 +281,7 @@ func (s *Server) enqueueKojiCompose(taskID uint64, server, name, version, releas
}
var ostreeResolveJobID uuid.UUID
commitSources := manifestSource.Content.OSTreeCommits
commitSources := manifestSource.GetOSTreeSourceSpecs()
if len(commitSources) > 1 {
// only one pipeline can specify an ostree commit for content
pipelines := make([]string, 0, len(commitSources))
@ -476,7 +476,7 @@ func serializeManifest(ctx context.Context, manifestSource *manifest.Manifest, w
// the container embedding, so we need to get it from the manifest
// content field. There should be only one.
var containerEmbedPipeline string
for name := range manifestSource.Content.Containers {
for name := range manifestSource.GetContainerSourceSpecs() {
containerEmbedPipeline = name
break
}
@ -519,7 +519,7 @@ func serializeManifest(ctx context.Context, manifestSource *manifest.Manifest, w
// ostree commits, so we need to get it from the manifest content
// field. There should be only one.
var ostreeCommitPipeline string
for name := range manifestSource.Content.OSTreeCommits {
for name := range manifestSource.GetOSTreeSourceSpecs() {
ostreeCommitPipeline = name
break
}

9
internal/distro/distro_test.go
View file

@ -61,7 +61,7 @@ func TestImageType_PackageSetsChains(t *testing.T) {
}
manifest, _, err := imageType.Manifest(&bp, options, nil, 0)
require.NoError(t, err)
imagePkgSets := manifest.Content.PackageSets
imagePkgSets := manifest.GetPackageSetChains()
for packageSetName := range imageType.PackageSetsChains() {
_, ok := imagePkgSets[packageSetName]
if !ok {
@ -168,8 +168,9 @@ func TestImageTypePipelineNames(t *testing.T) {
containers := make(map[string][]container.Spec, 0)
// "resolve" ostree commits by copying the source specs into commit specs
commits := make(map[string][]ostree.CommitSpec, len(m.Content.OSTreeCommits))
for name, commitSources := range m.Content.OSTreeCommits {
ostreeSources := m.GetOSTreeSourceSpecs()
commits := make(map[string][]ostree.CommitSpec, len(ostreeSources))
for name, commitSources := range ostreeSources {
commitSpecs := make([]ostree.CommitSpec, len(commitSources))
for idx, commitSource := range commitSources {
commitSpecs[idx] = ostree.CommitSpec{
@ -463,7 +464,7 @@ func TestPipelineRepositories(t *testing.T) {
repos := tCase.repos
manifest, _, err := imageType.Manifest(&bp, options, repos, 0)
require.NoError(err)
packageSets := manifest.Content.PackageSets
packageSets := manifest.GetPackageSetChains()
var globals stringSet
if len(tCase.result["*"]) > 0 {

4
internal/distro/distro_test_common/distro_test_common.go
View file

@ -172,7 +172,7 @@ func getImageTypePkgSpecSets(imageType distro.ImageType, bp blueprint.Blueprint,
if err != nil {
panic("Could not generate manifest for package sets: " + err.Error())
}
imgPackageSets := manifest.Content.PackageSets
imgPackageSets := manifest.GetPackageSetChains()
solver := dnfjson.NewSolver(imageType.Arch().Distro().ModulePlatformID(),
imageType.Arch().Distro().Releasever(),
@ -207,7 +207,7 @@ func kernelCount(imgType distro.ImageType, bp blueprint.Blueprint) int {
if err != nil {
panic(err)
}
sets := manifest.Content.PackageSets
sets := manifest.GetPackageSetChains()
// Use a map to count unique kernels in a package set. If the same kernel
// name appears twice, it will only be installed once, so we only count it

2
internal/distro/fedora/distro_test.go
View file

@ -252,7 +252,7 @@ func TestImageType_BuildPackages(t *testing.T) {
}
manifest, _, err := itStruct.Manifest(&blueprint.Blueprint{}, distro.ImageOptions{}, nil, 0)
assert.NoError(t, err)
buildPkgs := manifest.Content.PackageSets["build"]
buildPkgs := manifest.GetPackageSetChains()["build"]
assert.NotNil(t, buildPkgs)
assert.Len(t, buildPkgs, 1)
assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs[0].Include)

4
internal/distro/fedora/imagetype.go
View file

@ -230,10 +230,6 @@ func (t *imageType) Manifest(bp *blueprint.Blueprint,
return nil, nil, err
}
manifest.Content.PackageSets = manifest.GetPackageSetChains()
manifest.Content.Containers = manifest.GetContainerSourceSpecs()
manifest.Content.OSTreeCommits = manifest.GetOSTreeSourceSpecs()
return &manifest, warnings, err
}

2
internal/distro/rhel7/distro_test.go
View file

@ -119,7 +119,7 @@ func TestImageType_BuildPackages(t *testing.T) {
}
manifest, _, err := itStruct.Manifest(&blueprint.Blueprint{}, distro.ImageOptions{}, nil, 0)
assert.NoError(t, err)
buildPkgs := manifest.Content.PackageSets["build"]
buildPkgs := manifest.GetPackageSetChains()["build"]
assert.NotNil(t, buildPkgs)
assert.Len(t, buildPkgs, 1)
assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs[0].Include)

4
internal/distro/rhel7/imagetype.go
View file

@ -220,10 +220,6 @@ func (t *imageType) Manifest(bp *blueprint.Blueprint,
return nil, nil, err
}
manifest.Content.PackageSets = overridePackageNamesInSets(manifest.GetPackageSetChains())
manifest.Content.Containers = manifest.GetContainerSourceSpecs()
manifest.Content.OSTreeCommits = manifest.GetOSTreeSourceSpecs()
return &manifest, warnings, err
}

2
internal/distro/rhel8/distro_test.go
View file

@ -287,7 +287,7 @@ func TestImageType_BuildPackages(t *testing.T) {
}
manifest, _, err := itStruct.Manifest(&blueprint.Blueprint{}, distro.ImageOptions{}, nil, 0)
assert.NoError(t, err)
buildPkgs := manifest.Content.PackageSets["build"]
buildPkgs := manifest.GetPackageSetChains()["build"]
assert.NotNil(t, buildPkgs)
assert.Len(t, buildPkgs, 1)
assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs[0].Include)

4
internal/distro/rhel8/imagetype.go
View file

@ -254,10 +254,6 @@ func (t *imageType) Manifest(bp *blueprint.Blueprint,
return nil, nil, err
}
manifest.Content.PackageSets = overridePackageNamesInSets(manifest.GetPackageSetChains())
manifest.Content.Containers = manifest.GetContainerSourceSpecs()
manifest.Content.OSTreeCommits = manifest.GetOSTreeSourceSpecs()
return &manifest, warnings, err
}

2
internal/distro/rhel9/distro_test.go
View file

@ -280,7 +280,7 @@ func TestImageType_BuildPackages(t *testing.T) {
}
manifest, _, err := itStruct.Manifest(&blueprint.Blueprint{}, distro.ImageOptions{}, nil, 0)
assert.NoError(t, err)
buildPkgs := manifest.Content.PackageSets["build"]
buildPkgs := manifest.GetPackageSetChains()["build"]
assert.NotNil(t, buildPkgs)
assert.Len(t, buildPkgs, 1)
assert.ElementsMatch(t, buildPackages[archLabel], buildPkgs[0].Include)

4
internal/distro/rhel9/imagetype.go
View file

@ -254,10 +254,6 @@ func (t *imageType) Manifest(bp *blueprint.Blueprint,
return nil, nil, err
}
manifest.Content.PackageSets = manifest.GetPackageSetChains()
manifest.Content.Containers = manifest.GetContainerSourceSpecs()
manifest.Content.OSTreeCommits = manifest.GetOSTreeSourceSpecs()
return &manifest, warnings, err
}

4
internal/distro/test_distro/distro.go
View file

@ -284,10 +284,6 @@ func (t *TestImageType) Manifest(b *blueprint.Blueprint, options distro.ImageOpt
manifest.NewContentTest(m, buildPkgsKey, buildPackages, nil, nil)
manifest.NewContentTest(m, osPkgsKey, osPackages, nil, ostreeSources)
m.Content.PackageSets = m.GetPackageSetChains()
m.Content.Containers = m.GetContainerSourceSpecs()
m.Content.OSTreeCommits = m.GetOSTreeSourceSpecs()
return m, nil, nil
}

28
internal/manifest/manifest.go
View file

@ -48,34 +48,14 @@ func (m *OSBuildManifest) UnmarshalJSON(payload []byte) error {
}
// Manifest represents a manifest initialised with all the information required
// to generate the pipelines but no content. The content included in the
// Content field must be resolved before serializing.
// to generate the pipelines but no content. The content type sources
// (PackageSetChains, ContainerSourceSpecs, OSTreeSourceSpecs) must be
// retrieved through their corresponding Getters and resolved before
// serializing.
type Manifest struct {
// pipelines describe the build process for an image.
pipelines []Pipeline
// Content for the image that will be built by the Manifest. Each content
// type should be resolved before passing to the Serialize method.
Content Content
}
// Content for the image that will be built by the Manifest. Each content type
// should be resolved before passing to the Serialize method.
type Content struct {
// PackageSets are sequences of package sets, each set consisting of a list
// of package names to include and exclude and a set of repositories to use
// for resolving. Package set sequences (chains) should be depsolved in
// separately and the result combined. Package set sequences (chains) are
// keyed by the name of the Pipeline that will install them.
PackageSets map[string][]rpmmd.PackageSet
// Containers are source specifications for containers to embed in the image.
Containers map[string][]container.SourceSpec
// OSTreeCommits are source specifications for ostree commits to embed in
// the image or use as parent commits when building a new one.
OSTreeCommits map[string][]ostree.SourceSpec
}
func New() Manifest {

6
internal/weldr/api.go
View file

@ -2521,7 +2521,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
return
}
packageSets, err := api.depsolve(manifest.Content.PackageSets, imageType.Arch().Distro())
packageSets, err := api.depsolve(manifest.GetPackageSetChains(), imageType.Arch().Distro())
if err != nil {
errors := responseError{
ID: "DepsolveError",
@ -2531,7 +2531,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
return
}
containerSpecs, err := api.resolveContainers(manifest.Content.Containers)
containerSpecs, err := api.resolveContainers(manifest.GetContainerSourceSpecs())
if err != nil {
errors := responseError{
ID: "ContainerResolveError",
@ -2543,7 +2543,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
testMode := q.Get("test")
ostreeCommitSpecs, err := api.resolveOSTreeCommits(manifest.Content.OSTreeCommits, testMode == "1" || testMode == "2")
ostreeCommitSpecs, err := api.resolveOSTreeCommits(manifest.GetOSTreeSourceSpecs(), testMode == "1" || testMode == "2")
if err != nil {
errors := responseError{
ID: "OSTreeOptionsError",

10
internal/weldr/api_test.go
View file

@ -894,7 +894,7 @@ func TestCompose(t *testing.T) {
manifest, _, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, 0)
require.NoError(t, err)
rPkgs, rContainers, rCommits := test_distro.ResolveContent(manifest.Content.PackageSets, manifest.Content.Containers, manifest.Content.OSTreeCommits)
rPkgs, rContainers, rCommits := test_distro.ResolveContent(manifest.GetPackageSetChains(), manifest.GetContainerSourceSpecs(), manifest.GetOSTreeSourceSpecs())
mf, err := manifest.Serialize(rPkgs, rContainers, rCommits)
require.NoError(t, err)
@ -905,7 +905,7 @@ func TestCompose(t *testing.T) {
ostreeManifest, _, err := ostreeImgType.Manifest(nil, distro.ImageOptions{OSTree: &ostreeOptions}, nil, 0)
require.NoError(t, err)
rPkgs, rContainers, rCommits = test_distro.ResolveContent(ostreeManifest.Content.PackageSets, ostreeManifest.Content.Containers, ostreeManifest.Content.OSTreeCommits)
rPkgs, rContainers, rCommits = test_distro.ResolveContent(ostreeManifest.GetPackageSetChains(), ostreeManifest.GetContainerSourceSpecs(), ostreeManifest.GetOSTreeSourceSpecs())
omf, err := ostreeManifest.Serialize(rPkgs, rContainers, rCommits)
require.NoError(t, err)
@ -1013,7 +1013,7 @@ func TestCompose(t *testing.T) {
ostreeManifestOther, _, err := ostreeImgType.Manifest(nil, distro.ImageOptions{OSTree: &ostreeOptionsOther}, nil, 0)
require.NoError(t, err)
rPkgs, rContainers, rCommits = test_distro.ResolveContent(ostreeManifestOther.Content.PackageSets, ostreeManifestOther.Content.Containers, ostreeManifestOther.Content.OSTreeCommits)
rPkgs, rContainers, rCommits = test_distro.ResolveContent(ostreeManifestOther.GetPackageSetChains(), ostreeManifestOther.GetContainerSourceSpecs(), ostreeManifestOther.GetOSTreeSourceSpecs())
omfo, err := ostreeManifest.Serialize(rPkgs, rContainers, rCommits)
require.NoError(t, err)
@ -1053,7 +1053,7 @@ func TestCompose(t *testing.T) {
manifest2, _, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, 0)
require.NoError(t, err)
rPkgs, rContainers, rCommits = test_distro.ResolveContent(manifest2.Content.PackageSets, manifest2.Content.Containers, manifest2.Content.OSTreeCommits)
rPkgs, rContainers, rCommits = test_distro.ResolveContent(manifest2.GetPackageSetChains(), manifest2.GetContainerSourceSpecs(), manifest2.GetOSTreeSourceSpecs())
mf2, err := manifest2.Serialize(rPkgs, rContainers, rCommits)
require.NoError(t, err)
@ -2043,7 +2043,7 @@ func TestComposePOST_ImageTypeDenylist(t *testing.T) {
manifest, _, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, 0)
require.NoError(t, err)
rPkgs, rContainers, rCommits := test_distro.ResolveContent(manifest.Content.PackageSets, manifest.Content.Containers, manifest.Content.OSTreeCommits)
rPkgs, rContainers, rCommits := test_distro.ResolveContent(manifest.GetPackageSetChains(), manifest.GetContainerSourceSpecs(), manifest.GetOSTreeSourceSpecs())
mf, err := manifest.Serialize(rPkgs, rContainers, rCommits)
require.NoError(t, err)