particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

manifest: add resolved commit specs to Serialize()

Browse source 
Same as with package specs and container specs, the commit specs are
added to the manifest serialization after being resolved.
This commit is contained in:
Achilleas Koutsou 2023-05-31 16:55:21 +02:00 • committed by Ondřej Budai
parent e05d4b4a03
commit 89a398371d
21 changed files with 101 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

21
cmd/gen-manifests/main.go
View file

@ -181,7 +181,9 @@ func makeManifestJob(name string, imgType distro.ImageType, cr composeRequest, d
return fmt.Errorf("[%s] container resolution failed: %s", filename, err.Error())
}
mf, err := manifest.Serialize(packageSpecs, containerSpecs)
commitSpecs := resolvePipelineCommits(manifest.Content.OSTreeCommits)
mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs)
if err != nil {
return fmt.Errorf("[%s] manifest serialization failed: %s", filename, err.Error())
}
@ -279,6 +281,23 @@ func resolvePipelineContainers(containerSources map[string][]container.SourceSpe
return containerSpecs, nil
}
func resolvePipelineCommits(commitSources map[string][]ostree.SourceSpec) map[string][]ostree.CommitSpec {
// "resolve" ostree commits by copying the source specs into commit specs
commits := make(map[string][]ostree.CommitSpec, len(commitSources))
for name, commitSources := range commitSources {
commitSpecs := make([]ostree.CommitSpec, len(commitSources))
for idx, commitSource := range commitSources {
commitSpecs[idx] = ostree.CommitSpec{
Ref: commitSource.Ref,
URL: commitSource.URL,
Checksum: commitSource.Parent,
}
}
commits[name] = commitSpecs
}
return commits
}
func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) {
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir)
solver.SetDNFJSONPath("./dnf-json")

21
cmd/osbuild-pipeline/main.go
View file

@ -223,6 +223,20 @@ func main() {
containers[name] = containerSpecs
}
// "resolve" ostree commits by copying the source specs into commit specs
commits := make(map[string][]ostree.CommitSpec, len(manifest.Content.OSTreeCommits))
for name, commitSources := range manifest.Content.OSTreeCommits {
commitSpecs := make([]ostree.CommitSpec, len(commitSources))
for idx, commitSource := range commitSources {
commitSpecs[idx] = ostree.CommitSpec{
Ref: commitSource.Ref,
URL: commitSource.URL,
Checksum: commitSource.Parent,
}
}
commits[name] = commitSpecs
}
var bytes []byte
if rpmmdArg {
bytes, err = json.Marshal(depsolvedSets)
@ -230,12 +244,7 @@ func main() {
panic(err)
}
} else {
if composeRequest.OSTree.Ref == "" {
// use default OSTreeRef for image type
composeRequest.OSTree.Ref = imageType.OSTreeRef()
}
ms, err := manifest.Serialize(depsolvedSets, containers)
ms, err := manifest.Serialize(depsolvedSets, containers, commits)
if err != nil {
panic(err.Error())
}

2
cmd/osbuild-playground/playground.go
View file

@ -48,7 +48,7 @@ func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos
fmt.Fprintf(os.Stderr, "could not clean dnf cache: %s", err.Error())
}
bytes, err := manifest.Serialize(packageSpecs, nil)
bytes, err := manifest.Serialize(packageSpecs, nil, nil)
if err != nil {
panic("failed to serialize manifest: " + err.Error())
}

2
cmd/osbuild-store-dump/main.go
View file

@ -36,7 +36,7 @@ func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d di
pkgSpecSets[name] = res
}
mf, err := manifest.Serialize(pkgSpecSets, nil)
mf, err := manifest.Serialize(pkgSpecSets, nil, nil)
if err != nil {
panic(err)
}

3
internal/cloudapi/v2/server.go
View file

@ -507,7 +507,8 @@ func generateManifest(ctx context.Context, workers *worker.Server, depsolveJobID
} else {
panic(fmt.Sprintf("ImageType %q does not define payload pipelines - this is a programming error", imageType.Name()))
}
ms, err := manifest.Serialize(depsolveResults.PackageSpecs, map[string][]container.Spec{payloadPipelineName: containerSpecs})
// TODO: resolve ostree source spec from manifest content and pass here.
ms, err := manifest.Serialize(depsolveResults.PackageSpecs, map[string][]container.Spec{payloadPipelineName: containerSpecs}, nil)
jobResult.Manifest = ms
}

15
internal/distro/distro_test.go
View file

@ -171,7 +171,20 @@ func TestImageTypePipelineNames(t *testing.T) {
assert.NoError(err)
containers := make(map[string][]container.Spec, 0)
mf, err := m.Serialize(packageSets, containers)
// "resolve" ostree commits by copying the source specs into commit specs
commits := make(map[string][]ostree.CommitSpec, len(m.Content.OSTreeCommits))
for name, commitSources := range m.Content.OSTreeCommits {
commitSpecs := make([]ostree.CommitSpec, len(commitSources))
for idx, commitSource := range commitSources {
commitSpecs[idx] = ostree.CommitSpec{
Ref: commitSource.Ref,
URL: commitSource.URL,
Checksum: commitSource.Parent,
}
}
commits[name] = commitSpecs
}
mf, err := m.Serialize(packageSets, containers, commits)
assert.NoError(err)
pm := new(manifest)
err = json.Unmarshal(mf, pm)

16
internal/distro/distro_test_common/distro_test_common.go
View file

@ -149,7 +149,21 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis
t.Errorf("distro.Manifest() error = %v", err)
return
}
got, err := manifest.Serialize(imgPackageSpecSets, tt.Containers)
// "resolve" ostree commits by copying the source specs into commit specs
commits := make(map[string][]ostree.CommitSpec, len(manifest.Content.OSTreeCommits))
for name, commitSources := range manifest.Content.OSTreeCommits {
commitSpecs := make([]ostree.CommitSpec, len(commitSources))
for idx, commitSource := range commitSources {
commitSpecs[idx] = ostree.CommitSpec{
Ref: commitSource.Ref,
URL: commitSource.URL,
Checksum: commitSource.Parent,
}
}
commits[name] = commitSpecs
}
got, err := manifest.Serialize(imgPackageSpecSets, tt.Containers, commits)
if (err == nil && tt.Manifest == nil) || (err != nil && tt.Manifest != nil) {
t.Errorf("distro.Manifest() error = %v", err)

3
internal/manifest/anaconda.go
View file

@ -5,6 +5,7 @@ import (
"github.com/osbuild/osbuild-composer/internal/container"
"github.com/osbuild/osbuild-composer/internal/osbuild"
"github.com/osbuild/osbuild-composer/internal/ostree"
"github.com/osbuild/osbuild-composer/internal/platform"
"github.com/osbuild/osbuild-composer/internal/rpmmd"
"github.com/osbuild/osbuild-composer/internal/users"
@ -141,7 +142,7 @@ func (p *Anaconda) getPackageSpecs() []rpmmd.PackageSpec {
return p.packageSpecs
}
func (p *Anaconda) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec) {
func (p *Anaconda) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) {
if len(p.packageSpecs) > 0 {
panic("double call to serializeStart()")
}

3
internal/manifest/build.go
View file

@ -3,6 +3,7 @@ package manifest
import (
"github.com/osbuild/osbuild-composer/internal/container"
"github.com/osbuild/osbuild-composer/internal/osbuild"
"github.com/osbuild/osbuild-composer/internal/ostree"
"github.com/osbuild/osbuild-composer/internal/rpmmd"
"github.com/osbuild/osbuild-composer/internal/runner"
)
@ -68,7 +69,7 @@ func (p *Build) getPackageSpecs() []rpmmd.PackageSpec {
return p.packageSpecs
}
func (p *Build) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec) {
func (p *Build) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) {
if len(p.packageSpecs) > 0 {
panic("double call to serializeStart()")
}

3
internal/manifest/commit_server_tree.go
View file

@ -6,6 +6,7 @@ import (
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/container"
"github.com/osbuild/osbuild-composer/internal/osbuild"
"github.com/osbuild/osbuild-composer/internal/ostree"
"github.com/osbuild/osbuild-composer/internal/platform"
"github.com/osbuild/osbuild-composer/internal/rpmmd"
)
@ -83,7 +84,7 @@ func (p *OSTreeCommitServer) getPackageSpecs() []rpmmd.PackageSpec {
return p.packageSpecs
}
func (p *OSTreeCommitServer) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec) {
func (p *OSTreeCommitServer) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) {
if len(p.packageSpecs) > 0 {
panic("double call to serializeStart()")
}

3
internal/manifest/coreos_installer.go
View file

@ -7,6 +7,7 @@ import (
"github.com/osbuild/osbuild-composer/internal/fdo"
"github.com/osbuild/osbuild-composer/internal/ignition"
"github.com/osbuild/osbuild-composer/internal/osbuild"
"github.com/osbuild/osbuild-composer/internal/ostree"
"github.com/osbuild/osbuild-composer/internal/platform"
"github.com/osbuild/osbuild-composer/internal/rpmmd"
)
@ -122,7 +123,7 @@ func (p *CoreOSInstaller) getPackageSpecs() []rpmmd.PackageSpec {
return p.packageSpecs
}
func (p *CoreOSInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec) {
func (p *CoreOSInstaller) serializeStart(packages []rpmmd.PackageSpec, _ []container.Spec, _ []ostree.CommitSpec) {
if len(p.packageSpecs) > 0 {
panic("double call to serializeStart()")
}

4
internal/manifest/manifest.go
View file

@ -131,14 +131,14 @@ func (m Manifest) GetOSTreeSourceSpecs() map[string][]ostree.SourceSpec {
return ostreeSpecs
}
func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec) (OSBuildManifest, error) {
func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec) (OSBuildManifest, error) {
pipelines := make([]osbuild.Pipeline, 0)
packages := make([]rpmmd.PackageSpec, 0)
commits := make([]ostree.CommitSpec, 0)
inline := make([]string, 0)
containers := make([]container.Spec, 0)
for _, pipeline := range m.pipelines {
pipeline.serializeStart(packageSets[pipeline.Name()], containerSpecs[pipeline.Name()])
pipeline.serializeStart(packageSets[pipeline.Name()], containerSpecs[pipeline.Name()], ostreeCommits[pipeline.Name()])
}
for _, pipeline := range m.pipelines {
commits = append(commits, pipeline.getOSTreeCommits()...)

2
internal/manifest/os.go
View file

@ -291,7 +291,7 @@ func (p *OS) getContainerSpecs() []container.Spec {
return p.containerSpecs
}
func (p *OS) serializeStart(packages []rpmmd.PackageSpec, containers []container.Spec) {
func (p *OS) serializeStart(packages []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec) {
if len(p.packageSpecs) > 0 {
panic("double call to serializeStart()")
}

2
internal/manifest/os_test.go
View file

@ -30,7 +30,7 @@ func NewTestOS() *OS {
packages := []rpmmd.PackageSpec{
rpmmd.PackageSpec{Name: "pkg1"},
}
os.serializeStart(packages, nil)
os.serializeStart(packages, nil, nil)
return os
}

4
internal/manifest/pipeline.go
View file

@ -44,7 +44,7 @@ type Pipeline interface {
// its full Spec. See the ostree package for more details.
getOSTreeCommitSources() []ostree.SourceSpec
serializeStart([]rpmmd.PackageSpec, []container.Spec)
serializeStart([]rpmmd.PackageSpec, []container.Spec, []ostree.CommitSpec)
serializeEnd()
serialize() osbuild.Pipeline
@ -155,7 +155,7 @@ func NewBase(m *Manifest, name string, build *Build) Base {
// serializeStart must be called exactly once before each call
// to serialize().
func (p Base) serializeStart([]rpmmd.PackageSpec, []container.Spec) {
func (p Base) serializeStart([]rpmmd.PackageSpec, []container.Spec, []ostree.CommitSpec) {
}
// serializeEnd must be called exactly once after each call to

4
internal/store/fixtures.go
View file

@ -56,7 +56,7 @@ func FixtureBase() *Store {
panic(fmt.Sprintf("failed to create a manifest: %v", err))
}
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
if err != nil {
panic(fmt.Sprintf("failed to create a manifest: %v", err))
}
@ -198,7 +198,7 @@ func FixtureFinished() *Store {
panic(fmt.Sprintf("failed to create a manifest: %v", err))
}
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
if err != nil {
panic(fmt.Sprintf("failed to create a manifest: %v", err))
}

2
internal/store/store_test.go
View file

@ -53,7 +53,7 @@ func (suite *storeTest) SetupSuite() {
suite.myArch, _ = suite.myDistro.GetArch(test_distro.TestArchName)
suite.myImageType, _ = suite.myArch.GetImageType(test_distro.TestImageTypeName)
manifest, _, _ := suite.myImageType.Manifest(&suite.myBP, suite.myImageOptions, suite.myRepoConfig, 0)
suite.myManifest, _ = manifest.Serialize(nil, nil)
suite.myManifest, _ = manifest.Serialize(nil, nil, nil)
suite.mySourceConfig = SourceConfig{
Name: "testSourceConfig",
}

4
internal/weldr/api.go
View file

@ -2450,6 +2450,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
// Fake a parent commit for test requests
cr.OSTree.Parent = "02604b2da6e954bd34b8b82a835e5a77d2b60ffa"
} else if imageType.OSTreeRef() != "" {
// TODO: don't read image ref from image type directly; instead get it from the manifest after initialising.
// If the image type has a default ostree ref, assume this is an OSTree image
reqParams := cr.OSTree
if reqParams.Ref == "" {
@ -2533,7 +2534,8 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
return
}
mf, err := manifest.Serialize(packageSets, containerSpecs)
// TODO: resolve ostree source spec from manifest content and pass here.
mf, err := manifest.Serialize(packageSets, containerSpecs, nil)
if err != nil {
errors := responseError{
ID: "ManifestCreationFailed",

8
internal/weldr/api_test.go
View file

@ -886,7 +886,7 @@ func TestCompose(t *testing.T) {
manifest, _, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, 0)
require.NoError(t, err)
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
require.NoError(t, err)
ostreeImgType, err := arch.GetImageType(test_distro.TestImageTypeOSTree)
@ -894,7 +894,7 @@ func TestCompose(t *testing.T) {
ostreeManifest, _, err := ostreeImgType.Manifest(nil, distro.ImageOptions{}, nil, 0)
require.NoError(t, err)
omf, err := ostreeManifest.Serialize(nil, nil)
omf, err := ostreeManifest.Serialize(nil, nil, nil)
require.NoError(t, err)
expectedComposeLocal := &store.Compose{
@ -1004,7 +1004,7 @@ func TestCompose(t *testing.T) {
manifest2, _, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, 0)
require.NoError(t, err)
mf2, err := manifest2.Serialize(nil, nil)
mf2, err := manifest2.Serialize(nil, nil, nil)
require.NoError(t, err)
expectedComposeGoodDistro := &store.Compose{
@ -1998,7 +1998,7 @@ func TestComposePOST_ImageTypeDenylist(t *testing.T) {
manifest, _, err := imgType.Manifest(nil, distro.ImageOptions{}, nil, 0)
require.NoError(t, err)
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
require.NoError(t, err)
expectedComposeLocal := &store.Compose{

4
internal/weldr/compose_test.go
View file

@ -36,7 +36,7 @@ func TestComposeStatusFromLegacyError(t *testing.T) {
if err != nil {
t.Fatalf("error creating osbuild manifest: %v", err)
}
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
if err != nil {
t.Fatalf("error serializing osbuild manifest: %v", err)
}
@ -89,7 +89,7 @@ func TestComposeStatusFromJobError(t *testing.T) {
if err != nil {
t.Fatalf("error creating osbuild manifest: %v", err)
}
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
if err != nil {
t.Fatalf("error serializing osbuild manifest: %v", err)
}

14
internal/worker/server_test.go
View file

@ -135,7 +135,7 @@ func TestCreate(t *testing.T) {
if err != nil {
t.Fatalf("error creating osbuild manifest: %v", err)
}
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
if err != nil {
t.Fatalf("error creating osbuild manifest: %v", err)
}
@ -165,7 +165,7 @@ func TestCancel(t *testing.T) {
t.Fatalf("error creating osbuild manifest: %v", err)
}
server := newTestServer(t, t.TempDir(), time.Duration(0), "/api/worker/v1", false)
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
if err != nil {
t.Fatalf("error creating osbuild manifest: %v", err)
}
@ -206,7 +206,7 @@ func TestUpdate(t *testing.T) {
t.Fatalf("error creating osbuild manifest: %v", err)
}
server := newTestServer(t, t.TempDir(), time.Duration(0), "/api/worker/v1", false)
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
if err != nil {
t.Fatalf("error creating osbuild manifest: %v", err)
}
@ -238,7 +238,7 @@ func TestArgs(t *testing.T) {
manifest, _, err := imageType.Manifest(nil, distro.ImageOptions{Size: imageType.Size(0)}, nil, 0)
require.NoError(t, err)
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
require.NoError(t, err)
job := worker.OSBuildJob{
@ -289,7 +289,7 @@ func TestUpload(t *testing.T) {
t.Fatalf("error creating osbuild manifest: %v", err)
}
server := newTestServer(t, t.TempDir(), time.Duration(0), "/api/worker/v1", true)
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
if err != nil {
t.Fatalf("error creating osbuild manifest: %v", err)
}
@ -324,7 +324,7 @@ func TestUploadNotAcceptingArtifacts(t *testing.T) {
}
server := newTestServer(t, t.TempDir(), time.Duration(0), "/api/worker/v1", false)
handler := server.Handler()
mf, _ := manifest.Serialize(nil, nil)
mf, _ := manifest.Serialize(nil, nil, nil)
if err != nil {
t.Fatalf("error creating osbuild manifest: %v", err)
}
@ -356,7 +356,7 @@ func TestUploadAlteredBasePath(t *testing.T) {
if err != nil {
t.Fatalf("error creating osbuild manifest: %v", err)
}
mf, err := manifest.Serialize(nil, nil)
mf, err := manifest.Serialize(nil, nil, nil)
if err != nil {
t.Fatalf("error creating osbuild manifest: %v", err)
}