diff --git a/cmd/osbuild-dnf-json-tests/main_test.go b/cmd/osbuild-dnf-json-tests/main_test.go index fae5d29bd..885677a9b 100644 --- a/cmd/osbuild-dnf-json-tests/main_test.go +++ b/cmd/osbuild-dnf-json-tests/main_test.go @@ -15,6 +15,7 @@ import ( "github.com/osbuild/osbuild-composer/internal/blueprint" "github.com/osbuild/osbuild-composer/internal/distro" fedora "github.com/osbuild/osbuild-composer/internal/distro/fedora33" + rhel "github.com/osbuild/osbuild-composer/internal/distro/rhel86" "github.com/osbuild/osbuild-composer/internal/rpmmd" "github.com/osbuild/osbuild-composer/internal/test" ) @@ -94,3 +95,68 @@ func TestCrossArchDepsolve(t *testing.T) { }) } } + +// This test loads all the repositories available in /repositories directory +// and tries to depsolve all package sets of one image type for one architecture. +func TestDepsolvePackageSets(t *testing.T) { + // Load repositories from the definition we provide in the RPM package + repoDir := "/usr/share/tests/osbuild-composer" + + // NOTE: we can add RHEL, but don't make it hard requirement because it will fail outside of VPN + for _, distroStruct := range []distro.Distro{rhel.NewCentos()} { + t.Run(distroStruct.Name(), func(t *testing.T) { + + // Run tests in parallel to speed up run times. + t.Parallel() + + // Set up temporary directory for rpm/dnf cache + dir := t.TempDir() + + // use a fullpath to dnf-json, this allows this test to have an arbitrary + // working directory + rpm := rpmmd.NewRPMMD(dir) + + repos, err := rpmmd.LoadRepositories([]string{repoDir}, distroStruct.Name()) + require.NoErrorf(t, err, "Failed to LoadRepositories %v", distroStruct.Name()) + x86Repos, ok := repos[distro.X86_64ArchName] + require.Truef(t, ok, "failed to get %q repos for %q", distro.X86_64ArchName, distroStruct.Name()) + + x86Arch, err := distroStruct.GetArch(distro.X86_64ArchName) + require.Nilf(t, err, "failed to get %q arch of %q distro", distro.X86_64ArchName, distroStruct.Name()) + + qcow2ImageTypeName := "qcow2" + qcow2Image, err := x86Arch.GetImageType(qcow2ImageTypeName) + require.Nilf(t, err, "failed to get %q image type of %q/%q distro/arch", qcow2ImageTypeName, distroStruct.Name(), distro.X86_64ArchName) + + imagePkgSets := qcow2Image.PackageSets(blueprint.Blueprint{Packages: []blueprint.Package{{Name: "bind"}}}) + imagePkgSetChains := qcow2Image.PackageSetsChains() + require.NotEmptyf(t, imagePkgSetChains, "the %q image has no package set chains defined", qcow2ImageTypeName) + + expectedPackageSpecsSetNames := func(pkgSets map[string]rpmmd.PackageSet, pkgSetChains map[string][]string) []string { + expectedPkgSpecsSetNames := make([]string, 0, len(pkgSets)) + chainPkgSets := make(map[string]struct{}, len(pkgSets)) + for name, pkgSetChain := range pkgSetChains { + expectedPkgSpecsSetNames = append(expectedPkgSpecsSetNames, name) + for _, pkgSetName := range pkgSetChain { + chainPkgSets[pkgSetName] = struct{}{} + } + } + for name := range pkgSets { + if _, ok := chainPkgSets[name]; ok { + continue + } + expectedPkgSpecsSetNames = append(expectedPkgSpecsSetNames, name) + } + return expectedPkgSpecsSetNames + }(imagePkgSets, imagePkgSetChains) + + gotPackageSpecsSets, err := rpm.DepsolvePackageSets(imagePkgSetChains, imagePkgSets, x86Repos, nil, distroStruct.ModulePlatformID(), x86Arch.Name(), distroStruct.Releasever()) + require.Nil(t, err) + require.EqualValues(t, len(expectedPackageSpecsSetNames), len(gotPackageSpecsSets)) + for _, name := range expectedPackageSpecsSetNames { + _, ok := gotPackageSpecsSets[name] + assert.True(t, ok) + } + }) + } +} diff --git a/internal/mocks/rpmmd/fixtures.go b/internal/mocks/rpmmd/fixtures.go index abe7706ec..37817ebec 100644 --- a/internal/mocks/rpmmd/fixtures.go +++ b/internal/mocks/rpmmd/fixtures.go @@ -86,6 +86,57 @@ func createBaseDepsolveFixture() []rpmmd.PackageSpec { } } +func createBaseDepsolvePackageSetsFixture() map[string][]rpmmd.PackageSpec { + return map[string][]rpmmd.PackageSpec{ + "build": { + { + Name: "dep-package3", + Epoch: 7, + Version: "3.0.3", + Release: "1.fc30", + Arch: "x86_64", + }, + { + Name: "dep-package1", + Epoch: 0, + Version: "1.33", + Release: "2.fc30", + Arch: "x86_64", + }, + { + Name: "dep-package2", + Epoch: 0, + Version: "2.9", + Release: "1.fc30", + Arch: "x86_64", + }, + }, + "packages": { + { + Name: "dep-package3", + Epoch: 7, + Version: "3.0.3", + Release: "1.fc30", + Arch: "x86_64", + }, + { + Name: "dep-package1", + Epoch: 0, + Version: "1.33", + Release: "2.fc30", + Arch: "x86_64", + }, + { + Name: "dep-package2", + Epoch: 0, + Version: "2.9", + Release: "1.fc30", + Arch: "x86_64", + }, + }, + } +} + func BaseFixture(tmpdir string) Fixture { return Fixture{ fetchPackageList{ @@ -95,6 +146,7 @@ func BaseFixture(tmpdir string) Fixture { }, depsolve{ createBaseDepsolveFixture(), + createBaseDepsolvePackageSetsFixture(), map[string]string{"base": "sha256:f34848ca92665c342abd5816c9e3eda0e82180671195362bcd0080544a3bc2ac"}, nil, }, @@ -112,6 +164,7 @@ func NoComposesFixture(tmpdir string) Fixture { }, depsolve{ createBaseDepsolveFixture(), + createBaseDepsolvePackageSetsFixture(), map[string]string{"base": "sha256:f34848ca92665c342abd5816c9e3eda0e82180671195362bcd0080544a3bc2ac"}, nil, }, @@ -128,6 +181,7 @@ func NonExistingPackage(tmpdir string) Fixture { nil, }, depsolve{ + nil, nil, nil, &rpmmd.DNFError{ @@ -148,6 +202,7 @@ func BadDepsolve(tmpdir string) Fixture { nil, }, depsolve{ + nil, nil, nil, &rpmmd.DNFError{ @@ -171,6 +226,7 @@ func BadFetch(tmpdir string) Fixture { }, }, depsolve{ + nil, nil, nil, &rpmmd.DNFError{ @@ -192,6 +248,7 @@ func OldChangesFixture(tmpdir string) Fixture { }, depsolve{ createBaseDepsolveFixture(), + createBaseDepsolvePackageSetsFixture(), map[string]string{"base": "sha256:f34848ca92665c342abd5816c9e3eda0e82180671195362bcd0080544a3bc2ac"}, nil, }, diff --git a/internal/mocks/rpmmd/rpmmd_mock.go b/internal/mocks/rpmmd/rpmmd_mock.go index 89f524bbf..67a68d686 100644 --- a/internal/mocks/rpmmd/rpmmd_mock.go +++ b/internal/mocks/rpmmd/rpmmd_mock.go @@ -13,6 +13,7 @@ type fetchPackageList struct { } type depsolve struct { ret []rpmmd.PackageSpec + retSets map[string][]rpmmd.PackageSpec checksums map[string]string err error } @@ -39,3 +40,7 @@ func (r *rpmmdMock) FetchMetadata(repos []rpmmd.RepoConfig, modulePlatformID, ar func (r *rpmmdMock) Depsolve(packageSet rpmmd.PackageSet, repos []rpmmd.RepoConfig, modulePlatformID, arch, releasever string) ([]rpmmd.PackageSpec, map[string]string, error) { return r.Fixture.depsolve.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.depsolve.err } + +func (r *rpmmdMock) DepsolvePackageSets(packageSetsChains map[string][]string, packageSets map[string]rpmmd.PackageSet, repos []rpmmd.RepoConfig, packageSetsRepos map[string][]rpmmd.RepoConfig, modulePlatformID, arch, releasever string) (map[string][]rpmmd.PackageSpec, error) { + return r.Fixture.depsolve.retSets, r.Fixture.depsolve.err +} diff --git a/internal/rpmmd/repository.go b/internal/rpmmd/repository.go index a28013e3f..9ef18bca8 100644 --- a/internal/rpmmd/repository.go +++ b/internal/rpmmd/repository.go @@ -212,7 +212,15 @@ type RPMMD interface { // Depsolve takes a list of required content (specs), explicitly unwanted content (excludeSpecs), list // or repositories, and platform ID for modularity. It returns a list of all packages (with solved // dependencies) that will be installed into the system. + // + // NOTE: Kept for legacy purposes (Weldr). New code should default to using DepsolvePackageSets(). Depsolve(packageSet PackageSet, repos []RepoConfig, modulePlatformID, arch, releasever string) ([]PackageSpec, map[string]string, error) + + // DepsolvePackageSets takes a map of package sets chains, which should be depsolved as separate transactions, + // a map of package sets (included and excluded), a list of common and package-set-specific repositories, + // platform ID for modularity, architecture and release version. It returns a map of Package Specs, depsolved + // in a chain or alone, as defined based on the provided arguments. + DepsolvePackageSets(packageSetsChains map[string][]string, packageSets map[string]PackageSet, repos []RepoConfig, packageSetsRepos map[string][]RepoConfig, modulePlatformID, arch, releasever string) (map[string][]PackageSpec, error) } type DNFError struct { @@ -652,6 +660,55 @@ func (r *rpmmdImpl) chainDepsolve(chains []chainPackageSet, repos []RepoConfig, return dependencies, reply.Checksums, err } +// DepsolvePackageSets takes a map of package sets chains, which should be depsolved as separate transactions, +// a map of package sets (included and excluded), a list of common and package-set-specific repositories, +// platform ID for modularity, architecture and release version. It returns a map of Package Specs, depsolved +// in a chain or alone, as defined based on the provided arguments. +func (r *rpmmdImpl) DepsolvePackageSets( + packageSetsChains map[string][]string, + packageSets map[string]PackageSet, + repos []RepoConfig, + packageSetsRepos map[string][]RepoConfig, + modulePlatformID, arch, releasever string) (map[string][]PackageSpec, error) { + + packageSpecsSets := make(map[string][]PackageSpec, len(packageSets)) + // map to hold package set names which were processed as chains + chainPkgSets := make(map[string]struct{}, len(packageSets)) + + for name, packageSetChain := range packageSetsChains { + for _, packageSetName := range packageSetChain { + chainPkgSets[packageSetName] = struct{}{} + } + chainPkgSets, chainRepos, err := chainPackageSets(packageSetChain, packageSets, repos, packageSetsRepos) + if err != nil { + return nil, err + } + packageSpecs, _, err := r.chainDepsolve(chainPkgSets, chainRepos, modulePlatformID, arch, releasever) + if err != nil { + return nil, err + } + packageSpecsSets[name] = packageSpecs + } + + // Process the remaining package sets not contained in the package set chains + for name, packageSet := range packageSets { + if _, ok := chainPkgSets[name]; ok { + continue + } + chainPkgSets, chainRepos, err := chainPackageSets([]string{name}, map[string]PackageSet{name: packageSet}, repos, packageSetsRepos) + if err != nil { + return nil, err + } + packageSpecs, _, err := r.chainDepsolve(chainPkgSets, chainRepos, modulePlatformID, arch, releasever) + if err != nil { + return nil, err + } + packageSpecsSets[name] = packageSpecs + } + + return packageSpecsSets, nil +} + func (packages PackageList) Search(globPatterns ...string) (PackageList, error) { var globs []glob.Glob