upload/koji: don't depend on rpmmd.RPM for build metadata
The use of rpmmd.RPM is exclusive to the Koji upload implementation. Consolidate the metadata structure for RPMs into the koji package codebase, together with any required functions for converting osbuild stage metadata. The respective code in osbuild/images will be eventually removed. Signed-off-by: Tomáš Hozza <thozza@redhat.com>
This commit is contained in:
Tomáš Hozza
Tomáš Hozza
parent
0ca81d5af4
commit
0bafc0915d
6 changed files with 288 additions and 17 deletions
|
|
@ -24,7 +24,6 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/osbuild/images/pkg/platform"
|
||||
"github.com/osbuild/images/pkg/rpmmd"
|
||||
"github.com/osbuild/osbuild-composer/internal/upload/koji"
|
||||
)
|
||||
|
||||
|
|
@ -166,7 +165,7 @@ func TestKojiImport(t *testing.T) {
|
|||
Arch: "noarch",
|
||||
},
|
||||
Tools: []koji.Tool{},
|
||||
RPMs: []rpmmd.RPM{},
|
||||
RPMs: []koji.RPM{},
|
||||
},
|
||||
}
|
||||
output := []koji.BuildOutput{
|
||||
|
|
@ -178,7 +177,7 @@ func TestKojiImport(t *testing.T) {
|
|||
ChecksumType: koji.ChecksumTypeMD5,
|
||||
Checksum: hash,
|
||||
Type: koji.BuildOutputTypeImage,
|
||||
RPMs: []rpmmd.RPM{},
|
||||
RPMs: []koji.RPM{},
|
||||
Extra: &koji.BuildOutputExtra{
|
||||
ImageOutput: koji.ImageExtraInfo{
|
||||
Arch: "noarch",
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ import (
|
|||
|
||||
"github.com/google/uuid"
|
||||
"github.com/osbuild/images/pkg/platform"
|
||||
"github.com/osbuild/images/pkg/rpmmd"
|
||||
"github.com/osbuild/osbuild-composer/internal/upload/koji"
|
||||
)
|
||||
|
||||
|
|
@ -85,7 +84,7 @@ func main() {
|
|||
Arch: arch,
|
||||
},
|
||||
Tools: []koji.Tool{},
|
||||
RPMs: []rpmmd.RPM{},
|
||||
RPMs: []koji.RPM{},
|
||||
},
|
||||
}
|
||||
output := []koji.BuildOutput{
|
||||
|
|
@ -97,7 +96,7 @@ func main() {
|
|||
ChecksumType: koji.ChecksumTypeMD5,
|
||||
Checksum: hash,
|
||||
Type: koji.BuildOutputTypeImage,
|
||||
RPMs: []rpmmd.RPM{},
|
||||
RPMs: []koji.RPM{},
|
||||
Extra: &koji.BuildOutputExtra{
|
||||
ImageOutput: koji.ImageExtraInfo{
|
||||
Arch: arch,
|
||||
|
|
|
|||
|
|
@ -8,8 +8,6 @@ import (
|
|||
|
||||
"github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/osbuild/images/pkg/osbuild"
|
||||
"github.com/osbuild/images/pkg/rpmmd"
|
||||
"github.com/osbuild/osbuild-composer/internal/target"
|
||||
"github.com/osbuild/osbuild-composer/internal/upload/koji"
|
||||
"github.com/osbuild/osbuild-composer/internal/worker"
|
||||
|
|
@ -143,16 +141,16 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error {
|
|||
}
|
||||
|
||||
for i, buildResult := range osbuildResults {
|
||||
buildRPMs := make([]rpmmd.RPM, 0)
|
||||
buildRPMs := make([]koji.RPM, 0)
|
||||
// collect packages from stages in build pipelines
|
||||
for _, plName := range buildResult.PipelineNames.Build {
|
||||
buildPipelineMd := buildResult.OSBuildOutput.Metadata[plName]
|
||||
buildRPMs = append(buildRPMs, osbuild.OSBuildMetadataToRPMs(buildPipelineMd)...)
|
||||
buildRPMs = append(buildRPMs, koji.OSBuildMetadataToRPMs(buildPipelineMd)...)
|
||||
}
|
||||
// this dedupe is usually not necessary since we generally only have
|
||||
// one rpm stage in one build pipeline, but it's not invalid to have
|
||||
// multiple
|
||||
buildRPMs = rpmmd.DeduplicateRPMs(buildRPMs)
|
||||
buildRPMs = koji.DeduplicateRPMs(buildRPMs)
|
||||
|
||||
kojiTargetResults := buildResult.TargetResultsByName(target.TargetNameKoji)
|
||||
// Only a single Koji target is allowed per osbuild job
|
||||
|
|
@ -183,14 +181,14 @@ func (impl *KojiFinalizeJobImpl) Run(job worker.Job) error {
|
|||
})
|
||||
|
||||
// collect packages from stages in payload pipelines
|
||||
imageRPMs := make([]rpmmd.RPM, 0)
|
||||
imageRPMs := make([]koji.RPM, 0)
|
||||
for _, plName := range buildResult.PipelineNames.Payload {
|
||||
payloadPipelineMd := buildResult.OSBuildOutput.Metadata[plName]
|
||||
imageRPMs = append(imageRPMs, osbuild.OSBuildMetadataToRPMs(payloadPipelineMd)...)
|
||||
imageRPMs = append(imageRPMs, koji.OSBuildMetadataToRPMs(payloadPipelineMd)...)
|
||||
}
|
||||
|
||||
// deduplicate
|
||||
imageRPMs = rpmmd.DeduplicateRPMs(imageRPMs)
|
||||
imageRPMs = koji.DeduplicateRPMs(imageRPMs)
|
||||
|
||||
imgOutputExtraInfo := koji.ImageExtraInfo{
|
||||
Arch: buildResult.Arch,
|
||||
|
|
|
|||
68
internal/upload/koji/metadata-rpm.go
Normal file
68
internal/upload/koji/metadata-rpm.go
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
package koji
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/osbuild/images/pkg/osbuild"
|
||||
)
|
||||
|
||||
// RPM represents an RPM package in the Koji metadata format.
|
||||
// It contains the necessary fields to uniquely identify an RPM package,
|
||||
// when desdribing the build metadata in Koji.
|
||||
type RPM struct {
|
||||
Type string `json:"type"` // must be 'rpm'
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Release string `json:"release"`
|
||||
Epoch *string `json:"epoch,omitempty"`
|
||||
Arch string `json:"arch"`
|
||||
Sigmd5 string `json:"sigmd5"`
|
||||
Signature *string `json:"signature"`
|
||||
}
|
||||
|
||||
// NEVRA string for the package
|
||||
func (r RPM) String() string {
|
||||
epoch := ""
|
||||
if r.Epoch != nil {
|
||||
epoch = *r.Epoch + ":"
|
||||
}
|
||||
return fmt.Sprintf("%s-%s%s-%s.%s", r.Name, epoch, r.Version, r.Release, r.Arch)
|
||||
}
|
||||
|
||||
// Deduplicate a list of RPMs based on NEVRA string
|
||||
func DeduplicateRPMs(rpms []RPM) []RPM {
|
||||
rpmMap := make(map[string]struct{}, len(rpms))
|
||||
uniqueRPMs := make([]RPM, 0, len(rpms))
|
||||
|
||||
for _, rpm := range rpms {
|
||||
if _, added := rpmMap[rpm.String()]; !added {
|
||||
rpmMap[rpm.String()] = struct{}{}
|
||||
uniqueRPMs = append(uniqueRPMs, rpm)
|
||||
}
|
||||
}
|
||||
return uniqueRPMs
|
||||
}
|
||||
|
||||
func OSBuildMetadataToRPMs(stagesMetadata map[string]osbuild.StageMetadata) []RPM {
|
||||
rpms := make([]RPM, 0)
|
||||
for _, md := range stagesMetadata {
|
||||
switch metadata := md.(type) {
|
||||
case *osbuild.RPMStageMetadata:
|
||||
for _, pkg := range metadata.Packages {
|
||||
rpms = append(rpms, RPM{
|
||||
Type: "rpm",
|
||||
Name: pkg.Name,
|
||||
Epoch: pkg.Epoch,
|
||||
Version: pkg.Version,
|
||||
Release: pkg.Release,
|
||||
Arch: pkg.Arch,
|
||||
Sigmd5: pkg.SigMD5,
|
||||
Signature: osbuild.RPMPackageMetadataToSignature(pkg),
|
||||
})
|
||||
}
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
return rpms
|
||||
}
|
||||
208
internal/upload/koji/metadata-rpm_test.go
Normal file
208
internal/upload/koji/metadata-rpm_test.go
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
package koji
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/osbuild/images/pkg/osbuild"
|
||||
"github.com/osbuild/osbuild-composer/internal/common"
|
||||
)
|
||||
|
||||
func TestRPMDeduplication(t *testing.T) {
|
||||
require := require.New(t)
|
||||
// start with metadata, that includes duplicates, convert, then deduplicate
|
||||
metadata := osbuild.PipelineMetadata{
|
||||
"1": &osbuild.RPMStageMetadata{
|
||||
Packages: []osbuild.RPMPackageMetadata{
|
||||
// python38 twice
|
||||
{
|
||||
Name: "python38",
|
||||
Version: "3.8.8",
|
||||
Release: "4.module+el8.5.0+12205+a865257a",
|
||||
Epoch: nil,
|
||||
Arch: "x86_64",
|
||||
SigMD5: "-",
|
||||
SigPGP: "-",
|
||||
SigGPG: "-",
|
||||
},
|
||||
{
|
||||
Name: "python38",
|
||||
Version: "3.8.8",
|
||||
Release: "4.module+el8.5.0+12205+a865257a",
|
||||
Epoch: nil,
|
||||
Arch: "x86_64",
|
||||
SigMD5: "-",
|
||||
SigPGP: "-",
|
||||
SigGPG: "-",
|
||||
},
|
||||
// made up package
|
||||
{
|
||||
Name: "unique",
|
||||
Version: "1.90",
|
||||
Release: "10",
|
||||
Epoch: nil,
|
||||
Arch: "aarch64",
|
||||
SigMD5: ".",
|
||||
SigPGP: ".",
|
||||
SigGPG: ".",
|
||||
},
|
||||
// made up package with epoch
|
||||
{
|
||||
Name: "package-with-epoch",
|
||||
Version: "0.1",
|
||||
Release: "a",
|
||||
Epoch: common.ToPtr("8"),
|
||||
Arch: "x86_64",
|
||||
SigMD5: "*",
|
||||
SigPGP: "*",
|
||||
SigGPG: "*",
|
||||
},
|
||||
},
|
||||
},
|
||||
// separate pipeline
|
||||
"2": &osbuild.RPMStageMetadata{
|
||||
Packages: []osbuild.RPMPackageMetadata{
|
||||
// duplicate package with epoch
|
||||
{
|
||||
Name: "vim-minimal",
|
||||
Version: "8.0.1763",
|
||||
Release: "15.el8",
|
||||
Epoch: common.ToPtr("2"),
|
||||
Arch: "x86_64",
|
||||
SigMD5: "v",
|
||||
SigPGP: "v",
|
||||
SigGPG: "v",
|
||||
},
|
||||
{
|
||||
Name: "vim-minimal",
|
||||
Version: "8.0.1763",
|
||||
Release: "15.el8",
|
||||
Epoch: common.ToPtr("2"),
|
||||
Arch: "x86_64",
|
||||
SigMD5: "v",
|
||||
SigPGP: "v",
|
||||
SigGPG: "v",
|
||||
},
|
||||
// package with same name but different version
|
||||
{
|
||||
Name: "dupename",
|
||||
Version: "1",
|
||||
Release: "1.el8",
|
||||
Epoch: nil,
|
||||
Arch: "x86_64",
|
||||
SigMD5: "2",
|
||||
SigPGP: "2",
|
||||
SigGPG: "2",
|
||||
},
|
||||
{
|
||||
Name: "dupename",
|
||||
Version: "2",
|
||||
Release: "1.el8",
|
||||
Epoch: nil,
|
||||
Arch: "x86_64",
|
||||
SigMD5: "2",
|
||||
SigPGP: "2",
|
||||
SigGPG: "2",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
testNames := []string{"dupename", "dupename", "package-with-epoch", "python38", "python38", "unique", "vim-minimal", "vim-minimal"}
|
||||
testNamesDeduped := []string{"dupename", "dupename", "package-with-epoch", "python38", "unique", "vim-minimal"}
|
||||
|
||||
rpms := OSBuildMetadataToRPMs(metadata)
|
||||
|
||||
// basic sanity checks
|
||||
require.Len(rpms, 8)
|
||||
|
||||
sortedNames := func(rpms []RPM) []string {
|
||||
names := make([]string, len(rpms))
|
||||
for idx, rpm := range rpms {
|
||||
names[idx] = rpm.Name
|
||||
}
|
||||
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
names := sortedNames(rpms)
|
||||
require.Equal(names, testNames)
|
||||
|
||||
deduped := DeduplicateRPMs(rpms)
|
||||
require.Len(deduped, 6)
|
||||
dedupedNames := sortedNames(deduped)
|
||||
require.Equal(dedupedNames, testNamesDeduped)
|
||||
}
|
||||
|
||||
func Test_OSBuildMetadataToRPMs(t *testing.T) {
|
||||
raw := `
|
||||
{
|
||||
"org.osbuild.rpm": {
|
||||
"packages": [
|
||||
{
|
||||
"name": "python3-pyserial",
|
||||
"version": "3.4",
|
||||
"release": "7.fc32",
|
||||
"epoch": null,
|
||||
"arch": "noarch",
|
||||
"sigmd5": "378cb32f9f850b275ac4e04d21e8144b",
|
||||
"sigpgp": "89023304000108001d162104963a2beb02009608fe67ea4249fd77499570ff3105025f5a272b000a091049fd77499570ff31ccdb0ffe38b95a55ebf3c021526b3cd4f2358c7e23f7767d1f5ce4b7cccef7b33653c6a96a23022313a818fbaf7abeb41837910f0d3ac15664e02838d5939d38ff459aa0076e248728a032d3ae09ddfaec955f941601081a2e3f9bbd49586fd65c1bc1b31685aeb0405687d1791471eab7359ccf00d5584ddef680e99ebc8a4846316391b9baa68ac8ed8ad696ee16fd625d847f8edd92517df3ea6920a46b77b4f119715a0f619f38835d25e0bd0eb5cfad08cd9c796eace6a2b28f4d3dee552e6068255d9748dc2a1906c951e0ba8aed9922ab24e1f659413a06083f8a0bfea56cfff14bddef23bced449f36bcd369da72f90ddf0512e7b0801ba5a0c8eaa8eb0582c630815e992192042cfb0a7c7239f76219197c2fdf18b6553260c105280806d4f037d7b04bdf3da9fd7e9a207db5c71f7e548f4288928f047c989c4cb9cbb8088eec7bd2fa5c252e693f51a3cfc660f666af6a255a5ca0fd2216d5ccd66cbd9c11afa61067d7f615ec8d0dc0c879b5fe633d8c9443f97285da597e4da8a3993af36f0be06acfa9b8058ec70bbc78b876e4c6c5d2108fb05c15a74ba48a3d7ded697cbc1748c228d77d1e0794a41fd5240fa67c3ed745fe47555a47c3d6163d8ce95fd6c2d0d6fa48f8e5b411e571e442109b1cb200d9a8117ee08bfe645f96aca34f7b7559622bbab75143dcad59f126ae0d319e6668ebba417e725638c4febf2e",
|
||||
"siggpg": "883f0305005f2310139ec3e4c0f7e257e611023e11009f639c5fe64abaa76224dab3a9f70c2714a84c63bd009d1cc184fb4b428dfcd7c3556f4a5f860cc0187740"
|
||||
},
|
||||
{
|
||||
"name": "libgcc",
|
||||
"version": "10.0.1",
|
||||
"release": "0.11.fc32",
|
||||
"epoch": null,
|
||||
"arch": "x86_64",
|
||||
"sigmd5": "84fc907a5047aeebaf8da1642925a417",
|
||||
"sigpgp": "89023304000108001d162104963a2beb02009608fe67ea4249fd77499570ff3105025f5a272b000a091049fd77499570ff31ccdb0ffe38b95a55ebf3c021526b3cd4f2358c7e23f7767d1f5ce4b7cccef7b33653c6a96a23022313a818fbaf7abeb41837910f0d3ac15664e02838d5939d38ff459aa0076e248728a032d3ae09ddfaec955f941601081a2e3f9bbd49586fd65c1bc1b31685aeb0405687d1791471eab7359ccf00d5584ddef680e99ebc8a4846316391b9baa68ac8ed8ad696ee16fd625d847f8edd92517df3ea6920a46b77b4f119715a0f619f38835d25e0bd0eb5cfad08cd9c796eace6a2b28f4d3dee552e6068255d9748dc2a1906c951e0ba8aed9922ab24e1f659413a06083f8a0bfea56cfff14bddef23bced449f36bcd369da72f90ddf0512e7b0801ba5a0c8eaa8eb0582c630815e992192042cfb0a7c7239f76219197c2fdf18b6553260c105280806d4f037d7b04bdf3da9fd7e9a207db5c71f7e548f4288928f047c989c4cb9cbb8088eec7bd2fa5c252e693f51a3cfc660f666af6a255a5ca0fd2216d5ccd66cbd9c11afa61067d7f615ec8d0dc0c879b5fe633d8c9443f97285da597e4da8a3993af36f0be06acfa9b8058ec70bbc78b876e4c6c5d2108fb05c15a74ba48a3d7ded697cbc1748c228d77d1e0794a41fd5240fa67c3ed745fe47555a47c3d6163d8ce95fd6c2d0d6fa48f8e5b411e571e442109b1cb200d9a8117ee08bfe645f96aca34f7b7559622bbab75143dcad59f126ae0d319e6668ebba417e725638c4febf2e",
|
||||
"siggpg": null
|
||||
},
|
||||
{
|
||||
"name": "libgcc-madeup",
|
||||
"version": "10.0.1",
|
||||
"release": "0.11.fc32",
|
||||
"epoch": null,
|
||||
"arch": "x86_64",
|
||||
"sigmd5": "84fc907a5047aeebaf8da1642925a418",
|
||||
"sigpgp": null,
|
||||
"siggpg": null
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
metadata := new(osbuild.PipelineMetadata)
|
||||
err := json.Unmarshal([]byte(raw), metadata)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, metadata)
|
||||
require.Len(t, *metadata, 1)
|
||||
|
||||
rpms := OSBuildMetadataToRPMs(*metadata)
|
||||
|
||||
require.Len(t, rpms, 3)
|
||||
|
||||
signature1 := "89023304000108001d162104963a2beb02009608fe67ea4249fd77499570ff3105025f5a272b000a091049fd77499570ff31ccdb0ffe38b95a55ebf3c021526b3cd4f2358c7e23f7767d1f5ce4b7cccef7b33653c6a96a23022313a818fbaf7abeb41837910f0d3ac15664e02838d5939d38ff459aa0076e248728a032d3ae09ddfaec955f941601081a2e3f9bbd49586fd65c1bc1b31685aeb0405687d1791471eab7359ccf00d5584ddef680e99ebc8a4846316391b9baa68ac8ed8ad696ee16fd625d847f8edd92517df3ea6920a46b77b4f119715a0f619f38835d25e0bd0eb5cfad08cd9c796eace6a2b28f4d3dee552e6068255d9748dc2a1906c951e0ba8aed9922ab24e1f659413a06083f8a0bfea56cfff14bddef23bced449f36bcd369da72f90ddf0512e7b0801ba5a0c8eaa8eb0582c630815e992192042cfb0a7c7239f76219197c2fdf18b6553260c105280806d4f037d7b04bdf3da9fd7e9a207db5c71f7e548f4288928f047c989c4cb9cbb8088eec7bd2fa5c252e693f51a3cfc660f666af6a255a5ca0fd2216d5ccd66cbd9c11afa61067d7f615ec8d0dc0c879b5fe633d8c9443f97285da597e4da8a3993af36f0be06acfa9b8058ec70bbc78b876e4c6c5d2108fb05c15a74ba48a3d7ded697cbc1748c228d77d1e0794a41fd5240fa67c3ed745fe47555a47c3d6163d8ce95fd6c2d0d6fa48f8e5b411e571e442109b1cb200d9a8117ee08bfe645f96aca34f7b7559622bbab75143dcad59f126ae0d319e6668ebba417e725638c4febf2e"
|
||||
require.Equal(t, RPM{
|
||||
Type: "rpm",
|
||||
Name: "libgcc",
|
||||
Version: "10.0.1",
|
||||
Release: "0.11.fc32",
|
||||
Epoch: nil,
|
||||
Arch: "x86_64",
|
||||
Sigmd5: "84fc907a5047aeebaf8da1642925a417",
|
||||
Signature: &signature1,
|
||||
}, rpms[1])
|
||||
|
||||
// GPG has a priority over PGP
|
||||
signature0 := "883f0305005f2310139ec3e4c0f7e257e611023e11009f639c5fe64abaa76224dab3a9f70c2714a84c63bd009d1cc184fb4b428dfcd7c3556f4a5f860cc0187740"
|
||||
require.Equal(t, signature0, *rpms[0].Signature)
|
||||
|
||||
// if neither GPG nor PGP is set, the signature is nil
|
||||
require.Nil(t, rpms[2].Signature)
|
||||
}
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
package koji
|
||||
|
||||
import (
|
||||
"github.com/osbuild/images/pkg/rpmmd"
|
||||
"github.com/osbuild/osbuild-composer/internal/target"
|
||||
)
|
||||
|
||||
|
|
@ -76,7 +75,7 @@ type BuildRoot struct {
|
|||
ContentGenerator ContentGenerator `json:"content_generator"`
|
||||
Container Container `json:"container"`
|
||||
Tools []Tool `json:"tools"`
|
||||
RPMs []rpmmd.RPM `json:"components"`
|
||||
RPMs []RPM `json:"components"`
|
||||
}
|
||||
|
||||
// OUTPUT METADATA
|
||||
|
|
@ -174,7 +173,7 @@ type BuildOutput struct {
|
|||
ChecksumType ChecksumType `json:"checksum_type"`
|
||||
Checksum string `json:"checksum"`
|
||||
Type BuildOutputType `json:"type"`
|
||||
RPMs []rpmmd.RPM `json:"components,omitempty"`
|
||||
RPMs []RPM `json:"components,omitempty"`
|
||||
Extra *BuildOutputExtra `json:"extra,omitempty"`
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue