Update osbuild/images to v0.88.0

Adjust all paces that call `Solver.Depsolve()`, to cope with the changes
that enabled SBOM support.

Fix loading of testing repositories in the CloudAPI unit tests.

Signed-off-by: Tomáš Hozza <thozza@redhat.com>
This commit is contained in:
Tomáš Hozza 2024-09-06 18:21:15 +02:00 committed by Tomáš Hozza
parent b2096c2963
commit 7bdd036395
19 changed files with 169 additions and 57 deletions

View file

@ -6,6 +6,8 @@ linters-settings:
gomoddirectives:
replace-local: false
replace-allow-list:
- "github.com/osbuild/images"
linters:
enable:

View file

@ -26,6 +26,7 @@ import (
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/rhsm/facts"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/images/pkg/sbom"
"github.com/osbuild/images/pkg/dnfjson"
)
@ -326,12 +327,12 @@ func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d dist
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
repoConfigs := make(map[string][]rpmmd.RepoConfig)
for name, pkgSet := range packageSets {
res, repos, err := solver.Depsolve(pkgSet)
res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone)
if err != nil {
return nil, nil, err
}
depsolvedSets[name] = res
repoConfigs[name] = repos
depsolvedSets[name] = res.Packages
repoConfigs[name] = res.Repos
}
return depsolvedSets, repoConfigs, nil
}

View file

@ -18,6 +18,7 @@ import (
"github.com/osbuild/images/pkg/ostree"
"github.com/osbuild/images/pkg/reporegistry"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/images/pkg/sbom"
)
// This test loads all the repositories available in /repositories directory
@ -67,7 +68,7 @@ func TestCrossArchDepsolve(t *testing.T) {
assert.NoError(t, err)
for _, set := range manifest.GetPackageSetChains() {
_, _, err = solver.Depsolve(set)
_, err = solver.Depsolve(set, sbom.StandardTypeNone)
assert.NoError(t, err)
}
})
@ -109,12 +110,12 @@ func TestDepsolvePackageSets(t *testing.T) {
gotPackageSpecsSets := make(map[string][]rpmmd.PackageSpec, len(imagePkgSets))
gotRepoConfigs := make(map[string][]rpmmd.RepoConfig, len(imagePkgSets))
for name, pkgSet := range imagePkgSets {
res, repos, err := solver.Depsolve(pkgSet)
res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone)
if err != nil {
require.Nil(t, err)
}
gotPackageSpecsSets[name] = res
gotRepoConfigs[name] = repos
gotPackageSpecsSets[name] = res.Packages
gotRepoConfigs[name] = res.Repos
}
expectedPackageSpecsSetNames := []string{"build", "os"}
require.EqualValues(t, len(expectedPackageSpecsSetNames), len(gotPackageSpecsSets))

View file

@ -17,6 +17,7 @@ import (
"github.com/osbuild/images/pkg/manifest"
"github.com/osbuild/images/pkg/reporegistry"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/images/pkg/sbom"
"github.com/osbuild/osbuild-composer/internal/blueprint"
"github.com/osbuild/osbuild-composer/internal/store"
"github.com/osbuild/osbuild-composer/internal/target"
@ -32,12 +33,12 @@ func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d di
repoConfigs := make(map[string][]rpmmd.RepoConfig)
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), a.Name(), d.Name(), cacheDir)
for name, packages := range manifest.GetPackageSetChains() {
res, repos, err := solver.Depsolve(packages)
res, err := solver.Depsolve(packages, sbom.StandardTypeNone)
if err != nil {
panic(err)
}
pkgSpecSets[name] = res
repoConfigs[name] = repos
pkgSpecSets[name] = res.Packages
repoConfigs[name] = res.Repos
}
mf, err := manifest.Serialize(pkgSpecSets, nil, nil, repoConfigs)

View file

@ -10,6 +10,7 @@ import (
"github.com/osbuild/images/pkg/dnfjson"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/images/pkg/sbom"
"github.com/osbuild/osbuild-composer/internal/worker"
"github.com/osbuild/osbuild-composer/internal/worker/clienterrors"
)
@ -63,12 +64,12 @@ func (impl *DepsolveJobImpl) depsolve(packageSets map[string][]rpmmd.PackageSet,
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
repoConfigs := make(map[string][]rpmmd.RepoConfig)
for name, pkgSet := range packageSets {
res, repos, err := solver.Depsolve(pkgSet)
res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone)
if err != nil {
return nil, nil, err
}
depsolvedSets[name] = res
repoConfigs[name] = repos
depsolvedSets[name] = res.Packages
repoConfigs[name] = res.Repos
}
return depsolvedSets, repoConfigs, nil

2
go.mod
View file

@ -46,7 +46,7 @@ require (
github.com/labstack/gommon v0.4.2
github.com/openshift-online/ocm-sdk-go v0.1.438
github.com/oracle/oci-go-sdk/v54 v54.0.0
github.com/osbuild/images v0.87.0
github.com/osbuild/images v0.88.0
github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d
github.com/osbuild/pulp-client v0.1.0
github.com/prometheus/client_golang v1.20.2

4
go.sum
View file

@ -510,8 +510,8 @@ github.com/openshift-online/ocm-sdk-go v0.1.438 h1:tsLCCUzbLCTL4RZG02y9RuopmGCXp
github.com/openshift-online/ocm-sdk-go v0.1.438/go.mod h1:CiAu2jwl3ITKOxkeV0Qnhzv4gs35AmpIzVABQLtcI2Y=
github.com/oracle/oci-go-sdk/v54 v54.0.0 h1:CDLjeSejv2aDpElAJrhKpi6zvT/zhZCZuXchUUZ+LS4=
github.com/oracle/oci-go-sdk/v54 v54.0.0/go.mod h1:+t+yvcFGVp+3ZnztnyxqXfQDsMlq8U25faBLa+mqCMc=
github.com/osbuild/images v0.87.0 h1:ESCKlBhEeNaq9JB+Fo7JB/7KXmiQ5mgP0aoHAIQcI+4=
github.com/osbuild/images v0.87.0/go.mod h1:t+lyeUxOmLgyV+IZi4u+H+AyL1X59DffIgIb4KQXTGA=
github.com/osbuild/images v0.88.0 h1:0lehDocV1CoojGYQkRxn9sLx2Opwgaad7fluw4EzKmk=
github.com/osbuild/images v0.88.0/go.mod h1:t+lyeUxOmLgyV+IZi4u+H+AyL1X59DffIgIb4KQXTGA=
github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d h1:r9BFPDv0uuA9k1947Jybcxs36c/pTywWS1gjeizvtcQ=
github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d/go.mod h1:zR1iu/hOuf+OQNJlk70tju9IqzzM4ycq0ectkFBm94U=
github.com/osbuild/pulp-client v0.1.0 h1:L0C4ezBJGTamN3BKdv+rKLuq/WxXJbsFwz/Hj7aEmJ8=

View file

@ -35,7 +35,7 @@ func newV2Server(t *testing.T, dir string, depsolveChannels []string, enableJWT
distros := distrofactory.NewTestDefault()
require.NotNil(t, distros)
repos, err := reporegistry.NewTestedDefault()
repos, err := reporegistry.New([]string{"../../../test/data"})
require.Nil(t, err)
require.NotNil(t, repos)

View file

@ -42,6 +42,7 @@ import (
"github.com/osbuild/images/pkg/reporegistry"
"github.com/osbuild/images/pkg/rhsm/facts"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/images/pkg/sbom"
"github.com/osbuild/osbuild-composer/internal/blueprint"
"github.com/osbuild/osbuild-composer/internal/common"
"github.com/osbuild/osbuild-composer/internal/store"
@ -326,7 +327,7 @@ func (api *API) PreloadMetadata() {
}
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.hostArch, d.Name())
_, _, err = solver.Depsolve([]rpmmd.PackageSet{{Include: []string{"filesystem"}, Repositories: repos}})
_, err = solver.Depsolve([]rpmmd.PackageSet{{Include: []string{"filesystem"}, Repositories: repos}}, sbom.StandardTypeNone)
if err != nil {
log.Printf("Problem preloading distro metadata for %s: %s", distro, err)
}
@ -1290,7 +1291,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), archName, d.Name())
for i := range packageInfos {
pkgName := packageInfos[i].Name
solved, _, err := solver.Depsolve([]rpmmd.PackageSet{{Include: []string{pkgName}, Repositories: repos}})
res, err := solver.Depsolve([]rpmmd.PackageSet{{Include: []string{pkgName}, Repositories: repos}}, sbom.StandardTypeNone)
if err != nil {
errors := responseError{
ID: errorId,
@ -1299,7 +1300,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req
statusResponseError(writer, http.StatusBadRequest, errors)
return
}
packageInfos[i].Dependencies = solved
packageInfos[i].Dependencies = res.Packages
}
if err := solver.CleanCache(); err != nil {
// log and ignore
@ -1377,7 +1378,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt
}
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), archName, d.Name())
deps, _, err := solver.Depsolve([]rpmmd.PackageSet{{Include: names, Repositories: repos}})
res, err := solver.Depsolve([]rpmmd.PackageSet{{Include: names, Repositories: repos}}, sbom.StandardTypeNone)
if err != nil {
errors := responseError{
ID: "ProjectsError",
@ -1390,7 +1391,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt
// log and ignore
log.Printf("Error during rpm repo cache cleanup: %s", err.Error())
}
err = json.NewEncoder(writer).Encode(reply{Projects: deps})
err = json.NewEncoder(writer).Encode(reply{Projects: res.Packages})
common.PanicOnError(err)
}
@ -2294,12 +2295,13 @@ func (api *API) depsolve(packageSets map[string][]rpmmd.PackageSet, distroName s
repoConfigs := make(map[string][]rpmmd.RepoConfig)
for name, pkgSet := range packageSets {
res, repos, err := solver.Depsolve(pkgSet)
// TODO: SBOM support could be added here
res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone)
if err != nil {
return nil, nil, err
}
depsolvedSets[name] = res
repoConfigs[name] = repos
depsolvedSets[name] = res.Packages
repoConfigs[name] = res.Repos
}
if err := solver.CleanCache(); err != nil {
// log and ignore
@ -3611,7 +3613,7 @@ func (api *API) depsolveBlueprint(bp blueprint.Blueprint) ([]rpmmd.PackageSpec,
}
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), arch, d.Name())
solved, _, err := solver.Depsolve([]rpmmd.PackageSet{{Include: bp.GetPackages(), Repositories: repos}})
res, err := solver.Depsolve([]rpmmd.PackageSet{{Include: bp.GetPackages(), Repositories: repos}}, sbom.StandardTypeNone)
if err != nil {
return nil, err
}
@ -3620,7 +3622,7 @@ func (api *API) depsolveBlueprint(bp blueprint.Blueprint) ([]rpmmd.PackageSpec,
// log and ignore
log.Printf("Error during rpm repo cache cleanup: %s", err.Error())
}
return solved, nil
return res.Packages, nil
}
func (api *API) uploadsScheduleHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {

View file

@ -132,9 +132,6 @@ type ImageType interface {
// Returns the package set names safe to install custom packages via custom repositories.
PayloadPackageSets() []string
// Returns named arrays of package set names which should be depsolved in a chain.
PackageSetsChains() map[string][]string
// Returns the names of the stages that will produce the build output.
Exports() []string

View file

@ -118,10 +118,6 @@ func (t *imageType) PayloadPackageSets() []string {
return []string{blueprintPkgsKey}
}
func (t *imageType) PackageSetsChains() map[string][]string {
return make(map[string][]string)
}
func (t *imageType) Exports() []string {
if len(t.exports) > 0 {
return t.exports

View file

@ -182,10 +182,11 @@ func iotCommitPackageSet(t *imageType) rpmmd.PackageSet {
"podman-plugins", // deprecated in podman 5
},
})
} else {
}
if common.VersionGreaterThanOrEqual(t.arch.distro.osVersion, "41") {
ps = ps.Append(rpmmd.PackageSet{
Include: []string{
"bootupd", // added in F40+
"bootupd", // Added in F41+
},
})
}

View file

@ -161,10 +161,6 @@ func (t *ImageType) PayloadPackageSets() []string {
return []string{BlueprintPkgsKey}
}
func (t *ImageType) PackageSetsChains() map[string][]string {
return nil
}
func (t *ImageType) Exports() []string {
if len(t.exports) > 0 {
return t.exports

View file

@ -226,12 +226,6 @@ func (t *TestImageType) PayloadPackageSets() []string {
return []string{blueprintPkgsKey}
}
func (t *TestImageType) PackageSetsChains() map[string][]string {
return map[string][]string{
osPkgsKey: {osPkgsKey, blueprintPkgsKey},
}
}
func (t *TestImageType) Exports() []string {
return distro.ExportsFallback()
}

View file

@ -29,6 +29,7 @@ import (
"github.com/osbuild/images/internal/common"
"github.com/osbuild/images/pkg/rhsm"
"github.com/osbuild/images/pkg/rpmmd"
"github.com/osbuild/images/pkg/sbom"
)
// BaseSolver defines the basic solver configuration without platform
@ -155,6 +156,13 @@ type Solver struct {
subscriptions *rhsm.Subscriptions
}
// DepsolveResult contains the results of a depsolve operation.
type DepsolveResult struct {
Packages []rpmmd.PackageSpec
Repos []rpmmd.RepoConfig
SBOM *sbom.Document
}
// Create a new Solver with the given configuration. Initialising a Solver also loads system subscription information.
func NewSolver(modulePlatformID, releaseVer, arch, distro, cacheDir string) *Solver {
s := NewBaseSolver(cacheDir)
@ -193,10 +201,10 @@ func (s *Solver) SetProxy(proxy string) error {
// their associated repositories. Each package set is depsolved as a separate
// transactions in a chain. It returns a list of all packages (with solved
// dependencies) that will be installed into the system.
func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, []rpmmd.RepoConfig, error) {
req, rhsmMap, err := s.makeDepsolveRequest(pkgSets)
func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet, sbomType sbom.StandardType) (*DepsolveResult, error) {
req, rhsmMap, err := s.makeDepsolveRequest(pkgSets, sbomType)
if err != nil {
return nil, nil, fmt.Errorf("makeDepsolveRequest failed: %w", err)
return nil, fmt.Errorf("makeDepsolveRequest failed: %w", err)
}
// get non-exclusive read lock
@ -205,7 +213,7 @@ func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, []rp
output, err := run(s.dnfJsonCmd, req)
if err != nil {
return nil, nil, fmt.Errorf("running osbuild-depsolve-dnf failed:\n%w", err)
return nil, fmt.Errorf("running osbuild-depsolve-dnf failed:\n%w", err)
}
// touch repos to now
now := time.Now().Local()
@ -219,11 +227,24 @@ func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet) ([]rpmmd.PackageSpec, []rp
dec := json.NewDecoder(bytes.NewReader(output))
dec.DisallowUnknownFields()
if err := dec.Decode(&result); err != nil {
return nil, nil, fmt.Errorf("decoding depsolve result failed: %w", err)
return nil, fmt.Errorf("decoding depsolve result failed: %w", err)
}
packages, repos := result.toRPMMD(rhsmMap)
return packages, repos, nil
var sbomDoc *sbom.Document
if sbomType != sbom.StandardTypeNone {
sbomDoc, err = sbom.NewDocument(sbomType, result.SBOM)
if err != nil {
return nil, fmt.Errorf("creating SBOM document failed: %w", err)
}
}
return &DepsolveResult{
Packages: packages,
Repos: repos,
SBOM: sbomDoc,
}, nil
}
// FetchMetadata returns the list of all the available packages in repos and
@ -411,7 +432,7 @@ func (r *repoConfig) Hash() string {
// NOTE: Due to implementation limitations of DNF and dnf-json, each package set
// in the chain must use all of the repositories used by its predecessor.
// An error is returned if this requirement is not met.
func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[string]bool, error) {
func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet, sbomType sbom.StandardType) (*Request, map[string]bool, error) {
// dedupe repository configurations but maintain order
// the order in which repositories are added to the request affects the
// order of the dependencies in the result
@ -478,6 +499,10 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[
Arguments: args,
}
if sbomType != sbom.StandardTypeNone {
req.Arguments.Sbom = &sbomRequest{Type: sbomType.String()}
}
return &req, rhsmMap, nil
}
@ -642,6 +667,10 @@ func (r *Request) Hash() string {
return fmt.Sprintf("%x", h.Sum(nil))
}
type sbomRequest struct {
Type string `json:"type"`
}
// arguments for a dnf-json request
type arguments struct {
// Repositories to use for depsolving
@ -659,6 +688,9 @@ type arguments struct {
// Optional metadata to download for the repositories
OptionalMetadata []string `json:"optional-metadata,omitempty"`
// Optionally request an SBOM from depsolving
Sbom *sbomRequest `json:"sbom,omitempty"`
}
type searchArgs struct {
@ -692,7 +724,10 @@ type depsolveResult struct {
Repos map[string]repoConfig `json:"repos"`
// (optional) contains the solver used, e.g. "dnf5"
Solver string `json:"solver"`
Solver string `json:"solver,omitempty"`
// (optional) contains the SBOM for the depsolved transaction
SBOM json.RawMessage `json:"sbom,omitempty"`
}
// Package specification

View file

@ -152,7 +152,10 @@ func ResolveRef(location, ref string, consumerCerts bool, subs *rhsm.Subscriptio
if consumerCerts {
if subs == nil {
subs, err = rhsm.LoadSystemSubscriptions()
if subs.Consumer == nil || err != nil {
if err != nil {
return "", NewResolveRefError("error adding rhsm certificates when resolving ref: %s", err)
}
if subs.Consumer == nil {
return "", NewResolveRefError("error adding rhsm certificates when resolving ref")
}
}

View file

@ -2,6 +2,8 @@ package reporegistry
import (
"fmt"
"path/filepath"
"runtime"
"github.com/osbuild/images/pkg/distroidparser"
"github.com/osbuild/images/pkg/rpmmd"
@ -21,6 +23,9 @@ func New(repoConfigPaths []string) (*RepoRegistry, error) {
if err != nil {
return nil, err
}
if len(repositories) == 0 {
return nil, fmt.Errorf("no repositories found in the given paths: %v", repoConfigPaths)
}
return &RepoRegistry{repositories}, nil
}
@ -28,7 +33,13 @@ func New(repoConfigPaths []string) (*RepoRegistry, error) {
// NewTestedDefault returns a new RepoRegistry instance with the data
// loaded from the default test repositories
func NewTestedDefault() (*RepoRegistry, error) {
testReposPath := []string{"./test/data"}
_, callerSrc, _, ok := runtime.Caller(0)
var testReposPath []string
if !ok {
testReposPath = append(testReposPath, "../../test/data")
} else {
testReposPath = append(testReposPath, filepath.Join(filepath.Dir(callerSrc), "../../test/data"))
}
return New(testReposPath)
}

70
vendor/github.com/osbuild/images/pkg/sbom/document.go generated vendored Normal file
View file

@ -0,0 +1,70 @@
package sbom
import (
"encoding/json"
"fmt"
)
type StandardType uint64
const (
StandardTypeNone StandardType = iota
StandardTypeSpdx
)
func (t StandardType) String() string {
switch t {
case StandardTypeNone:
return "none"
case StandardTypeSpdx:
return "spdx"
default:
panic("invalid standard type")
}
}
func (t StandardType) MarshalJSON() ([]byte, error) {
var s string
if t == StandardTypeNone {
s = ""
} else {
s = t.String()
}
return json.Marshal(s)
}
func (t *StandardType) UnmarshalJSON(data []byte) error {
switch string(data) {
case `""`:
*t = StandardTypeNone
case `"spdx"`:
*t = StandardTypeSpdx
default:
return fmt.Errorf("invalid SBOM standard type: %s", data)
}
return nil
}
type Document struct {
// type of the document standard
DocType StandardType
// document in a specific standard JSON raw format
Document json.RawMessage
}
func NewDocument(docType StandardType, doc json.RawMessage) (*Document, error) {
switch docType {
case StandardTypeSpdx:
docType = StandardTypeSpdx
default:
return nil, fmt.Errorf("unsupported SBOM document type: %s", docType)
}
return &Document{
DocType: docType,
Document: doc,
}, nil
}

3
vendor/modules.txt vendored
View file

@ -947,7 +947,7 @@ github.com/oracle/oci-go-sdk/v54/identity
github.com/oracle/oci-go-sdk/v54/objectstorage
github.com/oracle/oci-go-sdk/v54/objectstorage/transfer
github.com/oracle/oci-go-sdk/v54/workrequests
# github.com/osbuild/images v0.87.0
# github.com/osbuild/images v0.88.0
## explicit; go 1.21.0
github.com/osbuild/images/internal/common
github.com/osbuild/images/internal/environment
@ -992,6 +992,7 @@ github.com/osbuild/images/pkg/rhsm
github.com/osbuild/images/pkg/rhsm/facts
github.com/osbuild/images/pkg/rpmmd
github.com/osbuild/images/pkg/runner
github.com/osbuild/images/pkg/sbom
# github.com/osbuild/osbuild-composer/pkg/splunk_logger v0.0.0-20240814102216-0239db53236d
## explicit; go 1.21
github.com/osbuild/osbuild-composer/pkg/splunk_logger