diff --git a/cmd/osbuild-dnf-json-tests/main_test.go b/cmd/osbuild-dnf-json-tests/main_test.go index b0e0d90f0..ea81a02e9 100644 --- a/cmd/osbuild-dnf-json-tests/main_test.go +++ b/cmd/osbuild-dnf-json-tests/main_test.go @@ -83,10 +83,10 @@ func TestCrossArchDepsolve(t *testing.T) { packages := imgType.PackageSets(blueprint.Blueprint{}) - _, err = solver.Depsolve([]rpmmd.PackageSet{packages["build"]}, repos[archStr], nil) + _, err = solver.Depsolve([]rpmmd.PackageSet{packages["build"]}, repos[archStr]) assert.NoError(t, err) - _, err = solver.Depsolve([]rpmmd.PackageSet{packages["packages"]}, repos[archStr], nil) + _, err = solver.Depsolve([]rpmmd.PackageSet{packages["packages"]}, repos[archStr]) assert.NoError(t, err) }) } @@ -155,7 +155,7 @@ func TestDepsolvePackageSets(t *testing.T) { pkgSets[idx] = imagePkgSets[pkgSetName] delete(imagePkgSets, pkgSetName) // will be depsolved here: remove from map } - res, err := solver.Depsolve(pkgSets, x86Repos, nil) + res, err := solver.Depsolve(pkgSets, x86Repos) if err != nil { require.Nil(t, err) } @@ -164,7 +164,7 @@ func TestDepsolvePackageSets(t *testing.T) { // depsolve the rest of the package sets for name, pkgSet := range imagePkgSets { - res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, x86Repos, nil) + res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, x86Repos) if err != nil { require.Nil(t, err) } diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go index 61633d0ed..cdddbf8a1 100644 --- a/cmd/osbuild-pipeline/main.go +++ b/cmd/osbuild-pipeline/main.go @@ -152,7 +152,7 @@ func main() { pkgSets[idx] = packageSets[pkgSetName] delete(packageSets, pkgSetName) // will be depsolved here: remove from map } - res, err := solver.Depsolve(pkgSets, repos, nil) + res, err := solver.Depsolve(pkgSets, repos) if err != nil { panic("Could not depsolve: " + err.Error()) } @@ -161,7 +161,7 @@ func main() { // depsolve the rest of the package sets for name, pkgSet := range packageSets { - res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, repos, nil) + res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, repos) if err != nil { panic("Could not depsolve: " + err.Error()) } diff --git a/cmd/osbuild-store-dump/main.go b/cmd/osbuild-store-dump/main.go index 67832fc0d..244f9617a 100644 --- a/cmd/osbuild-store-dump/main.go +++ b/cmd/osbuild-store-dump/main.go @@ -23,7 +23,7 @@ func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d di pkgSpecSets := make(map[string][]rpmmd.PackageSpec) solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), a.Name(), cacheDir) for name, packages := range packageSets { - res, err := solver.Depsolve([]rpmmd.PackageSet{packages}, repos, nil) + res, err := solver.Depsolve([]rpmmd.PackageSet{packages}, repos) if err != nil { panic(err) } diff --git a/cmd/osbuild-worker/jobimpl-depsolve.go b/cmd/osbuild-worker/jobimpl-depsolve.go index b2eeede23..b35a3739c 100644 --- a/cmd/osbuild-worker/jobimpl-depsolve.go +++ b/cmd/osbuild-worker/jobimpl-depsolve.go @@ -21,18 +21,17 @@ type DepsolveJobImpl struct { // (matching map keys). func (impl *DepsolveJobImpl) depsolve(packageSetsChains map[string][]string, packageSets map[string]rpmmd.PackageSet, repos []rpmmd.RepoConfig, packageSetsRepos map[string][]rpmmd.RepoConfig, modulePlatformID, arch, releasever string) (map[string][]rpmmd.PackageSpec, error) { solver := dnfjson.NewSolver(modulePlatformID, releasever, arch, impl.RPMMDCache) - depsolvedSets := make(map[string][]rpmmd.PackageSpec) - psRepos := make([][]rpmmd.RepoConfig, 0) + depsolvedSets := make(map[string][]rpmmd.PackageSpec) // first depsolve package sets that are part of a chain for specName, setNames := range packageSetsChains { pkgSets := make([]rpmmd.PackageSet, len(setNames)) for idx, pkgSetName := range setNames { pkgSets[idx] = packageSets[pkgSetName] - psRepos = append(psRepos, packageSetsRepos[pkgSetName]) // will be nil if it doesn't exist - delete(packageSets, pkgSetName) // will be depsolved here: remove from map + pkgSets[idx].Repositories = packageSetsRepos[pkgSetName] + delete(packageSets, pkgSetName) // will be depsolved here: remove from map } - res, err := solver.Depsolve(pkgSets, repos, psRepos) + res, err := solver.Depsolve(pkgSets, repos) if err != nil { return nil, err } @@ -41,7 +40,8 @@ func (impl *DepsolveJobImpl) depsolve(packageSetsChains map[string][]string, pac // depsolve the rest of the package sets for name, pkgSet := range packageSets { - res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, repos, [][]rpmmd.RepoConfig{packageSetsRepos[name]}) + pkgSet.Repositories = packageSetsRepos[name] + res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, repos) if err != nil { return nil, err } diff --git a/internal/distro/distro_test_common/distro_test_common.go b/internal/distro/distro_test_common/distro_test_common.go index 5e41ab052..90de43d6f 100644 --- a/internal/distro/distro_test_common/distro_test_common.go +++ b/internal/distro/distro_test_common/distro_test_common.go @@ -143,7 +143,7 @@ func getImageTypePkgSpecSets(imageType distro.ImageType, bp blueprint.Blueprint, solver := dnfjson.NewSolver(imageType.Arch().Distro().ModulePlatformID(), imageType.Arch().Distro().Releasever(), imageType.Arch().Name(), cacheDir) imgPackageSpecSets := make(map[string][]rpmmd.PackageSpec) for name, packages := range imgPackageSets { - res, err := solver.Depsolve([]rpmmd.PackageSet{packages}, repos, nil) + res, err := solver.Depsolve([]rpmmd.PackageSet{packages}, repos) if err != nil { panic("Could not depsolve: " + err.Error()) } diff --git a/internal/dnfjson/dnfjson.go b/internal/dnfjson/dnfjson.go index 60115e068..c225cc0fb 100644 --- a/internal/dnfjson/dnfjson.go +++ b/internal/dnfjson/dnfjson.go @@ -87,16 +87,16 @@ func NewSolver(modulePlatformID string, releaseVer string, arch string, cacheDir } // Depsolve the given packages with explicit excludes using the given configuration and repos -func Depsolve(pkgSets []rpmmd.PackageSet, repos []rpmmd.RepoConfig, psRepos [][]rpmmd.RepoConfig, modulePlatformID string, releaseVer string, arch string, cacheDir string) (*DepsolveResult, error) { - return NewSolver(modulePlatformID, releaseVer, arch, cacheDir).Depsolve(pkgSets, repos, psRepos) +func Depsolve(pkgSets []rpmmd.PackageSet, repos []rpmmd.RepoConfig, modulePlatformID string, releaseVer string, arch string, cacheDir string) (*DepsolveResult, error) { + return NewSolver(modulePlatformID, releaseVer, arch, cacheDir).Depsolve(pkgSets, repos) } // Depsolve the list of required package sets with explicit excludes using // the given repositories. Each package set is depsolved as a separate // transactions in a chain. It returns a list of all packages (with solved // dependencies) that will be installed into the system. -func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet, repos []rpmmd.RepoConfig, psRepos [][]rpmmd.RepoConfig) (*DepsolveResult, error) { - req, repoMap, err := s.makeDepsolveRequest(pkgSets, repos, psRepos) +func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet, baseRepos []rpmmd.RepoConfig) (*DepsolveResult, error) { + req, repoMap, err := s.makeDepsolveRequest(pkgSets, baseRepos) if err != nil { return nil, err } @@ -205,11 +205,7 @@ type repoConfig struct { // NOTE: Due to implementation limitations of DNF and dnf-json, each package set // in the chain must use all of the repositories used by its predecessor. // An error is returned if this requirement is not met. -func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet, repoConfigs []rpmmd.RepoConfig, pkgsetsRepos [][]rpmmd.RepoConfig) (*Request, map[string]rpmmd.RepoConfig, error) { - // pkgsetsRepos must either be nil (empty) or the same length as the pkgSets array - if len(pkgsetsRepos) > 0 && len(pkgSets) != len(pkgsetsRepos) { - return nil, nil, fmt.Errorf("depsolve: the number of package set repository configurations (%d) does not match the number of package sets (%d)", len(pkgsetsRepos), len(pkgSets)) - } +func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet, baseRepos []rpmmd.RepoConfig) (*Request, map[string]rpmmd.RepoConfig, error) { // dedupe repository configurations but maintain order // the order in which repositories are added to the request affects the @@ -218,17 +214,15 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet, repoConfigs []r rpmRepoMap := make(map[string]rpmmd.RepoConfig) // These repo IDs will be used for all transactions in the chain - baseRepoIDs := make([]string, len(repoConfigs)) - for idx, repo := range repoConfigs { + baseRepoIDs := make([]string, len(baseRepos)) + for idx, repo := range baseRepos { id := repo.Hash() rpmRepoMap[id] = repo baseRepoIDs[idx] = id repos = append(repos, repo) } - - // extra repositories defined for specific package sets - for _, jobRepos := range pkgsetsRepos { - for _, repo := range jobRepos { + for _, ps := range pkgSets { + for _, repo := range ps.Repositories { id := repo.Hash() if _, ok := rpmRepoMap[id]; !ok { rpmRepoMap[id] = repo @@ -245,12 +239,7 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet, repoConfigs []r RepoIDs: baseRepoIDs, // due to its capacity, the slice will be copied when appended to } - if len(pkgsetsRepos) == 0 { - // nothing to do - continue - } - - for _, jobRepo := range pkgsetsRepos[dsIdx] { + for _, jobRepo := range pkgSet.Repositories { transactions[dsIdx].RepoIDs = append(transactions[dsIdx].RepoIDs, jobRepo.Hash()) } diff --git a/internal/dnfjson/dnfjson_test.go b/internal/dnfjson/dnfjson_test.go index 95debf6c6..bdd3b76d0 100644 --- a/internal/dnfjson/dnfjson_test.go +++ b/internal/dnfjson/dnfjson_test.go @@ -22,7 +22,7 @@ func TestDepsolver(t *testing.T) { { // single depsolve pkgsets := []rpmmd.PackageSet{{Include: []string{"kernel", "vim-minimal", "tmux", "zsh"}}} // everything you'll ever need - deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig}, nil) + deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig}) if err != nil { t.Fatal(err) } @@ -35,7 +35,7 @@ func TestDepsolver(t *testing.T) { {Include: []string{"kernel"}}, {Include: []string{"vim-minimal", "tmux", "zsh"}}, } - deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig}, nil) + deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig}) if err != nil { t.Fatal(err) } @@ -48,7 +48,8 @@ func TestDepsolver(t *testing.T) { {Include: []string{"kernel"}}, {Include: []string{"vim-minimal", "tmux", "zsh"}}, } - deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig}, [][]rpmmd.RepoConfig{nil, {s.RepoConfig}}) + pkgsets[1].Repositories = []rpmmd.RepoConfig{s.RepoConfig} + deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig}) if err != nil { t.Fatal(err) } @@ -123,11 +124,11 @@ func TestMakeDepsolveRequest(t *testing.T) { Exclude: []string{"pkg2"}, }, { - Include: []string{"pkg3"}, + Include: []string{"pkg3"}, + Repositories: []rpmmd.RepoConfig{userRepo}, }, }, - repos: []rpmmd.RepoConfig{baseOS, appstream}, - packageSetsRepos: [][]rpmmd.RepoConfig{nil, {userRepo}}, + repos: []rpmmd.RepoConfig{baseOS, appstream}, args: []transactionArgs{ { PackageSpecs: []string{"pkg1"}, @@ -201,14 +202,15 @@ func TestMakeDepsolveRequest(t *testing.T) { Exclude: []string{"pkg2"}, }, { - Include: []string{"pkg3"}, + Include: []string{"pkg3"}, + Repositories: []rpmmd.RepoConfig{userRepo}, }, { - Include: []string{"pkg4"}, + Include: []string{"pkg4"}, + Repositories: []rpmmd.RepoConfig{userRepo}, }, }, - repos: []rpmmd.RepoConfig{baseOS, appstream}, - packageSetsRepos: [][]rpmmd.RepoConfig{nil, {userRepo}, {userRepo}}, + repos: []rpmmd.RepoConfig{baseOS, appstream}, args: []transactionArgs{ { PackageSpecs: []string{"pkg1"}, @@ -251,14 +253,15 @@ func TestMakeDepsolveRequest(t *testing.T) { Exclude: []string{"pkg2"}, }, { - Include: []string{"pkg3"}, + Include: []string{"pkg3"}, + Repositories: []rpmmd.RepoConfig{userRepo}, }, { - Include: []string{"pkg4"}, + Include: []string{"pkg4"}, + Repositories: []rpmmd.RepoConfig{userRepo, userRepo2}, }, }, - repos: []rpmmd.RepoConfig{baseOS, appstream}, - packageSetsRepos: [][]rpmmd.RepoConfig{nil, {userRepo}, {userRepo, userRepo2}}, + repos: []rpmmd.RepoConfig{baseOS, appstream}, args: []transactionArgs{ { PackageSpecs: []string{"pkg1"}, @@ -305,17 +308,18 @@ func TestMakeDepsolveRequest(t *testing.T) { Exclude: []string{"pkg2"}, }, { - Include: []string{"pkg3"}, + Include: []string{"pkg3"}, + Repositories: []rpmmd.RepoConfig{userRepo}, }, { - Include: []string{"pkg4"}, + Include: []string{"pkg4"}, + Repositories: []rpmmd.RepoConfig{userRepo2}, }, }, - repos: []rpmmd.RepoConfig{baseOS, appstream}, - packageSetsRepos: [][]rpmmd.RepoConfig{nil, {userRepo}, {userRepo2}}, - err: true, + repos: []rpmmd.RepoConfig{baseOS, appstream}, + err: true, }, - // Error: 3 transactions but only 1 packageSetsRepos + // Error: 3 transactions but last one doesn't specify user repos in 2nd { packageSets: []rpmmd.PackageSet{ { @@ -323,21 +327,21 @@ func TestMakeDepsolveRequest(t *testing.T) { Exclude: []string{"pkg2"}, }, { - Include: []string{"pkg3"}, + Include: []string{"pkg3"}, + Repositories: []rpmmd.RepoConfig{userRepo, userRepo2}, }, { Include: []string{"pkg4"}, }, }, - repos: []rpmmd.RepoConfig{baseOS, appstream}, - packageSetsRepos: [][]rpmmd.RepoConfig{{userRepo, userRepo2}}, - err: true, + repos: []rpmmd.RepoConfig{baseOS, appstream}, + err: true, }, } solver := NewSolver("", "", "", "") for idx, tt := range tests { t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { - req, _, err := solver.makeDepsolveRequest(tt.packageSets, tt.repos, tt.packageSetsRepos) + req, _, err := solver.makeDepsolveRequest(tt.packageSets, tt.repos) if tt.err { assert.NotNilf(t, err, "expected an error, but got 'nil' instead") assert.Nilf(t, req, "got non-nill request, but expected an error") diff --git a/internal/kojiapi/server.go b/internal/kojiapi/server.go index 35e809adc..3762b58d5 100644 --- a/internal/kojiapi/server.go +++ b/internal/kojiapi/server.go @@ -131,7 +131,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error { for idx, pkgSetName := range setNames { chain[idx] = packageSets[pkgSetName] } - res, err := solver.Depsolve(chain, repositories, nil) + res, err := solver.Depsolve(chain, repositories) if err != nil { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to depsolve base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err)) } diff --git a/internal/rpmmd/repository.go b/internal/rpmmd/repository.go index e42c52640..4cf737bc9 100644 --- a/internal/rpmmd/repository.go +++ b/internal/rpmmd/repository.go @@ -97,11 +97,13 @@ func (pkg Package) ToPackageInfo() PackageInfo { } } -// The inputs to depsolve, a set of packages to include and a set of -// packages to exclude. +// The inputs to depsolve, a set of packages to include and a set of packages +// to exclude. The Repositories are used when depsolving this package set in +// addition to the base repositories. type PackageSet struct { - Include []string - Exclude []string + Include []string + Exclude []string + Repositories []RepoConfig } // Append the Include and Exclude package list from another PackageSet and diff --git a/internal/weldr/api.go b/internal/weldr/api.go index b3a340bd9..c426a6465 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -1261,7 +1261,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName) for i := range packageInfos { pkgName := packageInfos[i].Name - solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: []string{pkgName}}}, repos, nil) + solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: []string{pkgName}}}, repos) if err != nil { errors := responseError{ ID: errorId, @@ -1339,11 +1339,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt } solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName) - deps, err := solver.Depsolve( - []rpmmd.PackageSet{{Include: names}}, - repos, - nil, - ) + deps, err := solver.Depsolve([]rpmmd.PackageSet{{Include: names}}, repos) if err != nil { errors := responseError{ ID: "ProjectsError", @@ -2145,23 +2141,23 @@ func (api *API) depsolveBlueprintForImageType(bp blueprint.Blueprint, imageType packageSetsRepos[name] = payloadRepos } - packageSets := imageType.PackageSets(bp) - depsolvedSets := make(map[string][]rpmmd.PackageSpec) - platformID := imageType.Arch().Distro().ModulePlatformID() releasever := imageType.Arch().Distro().Releasever() solver := api.solver.NewWithConfig(platformID, releasever, api.archName) - psRepos := make([][]rpmmd.RepoConfig, 0) + packageSets := imageType.PackageSets(bp) + depsolvedSets := make(map[string][]rpmmd.PackageSpec) // first depsolve package sets that are part of a chain for specName, setNames := range imageType.PackageSetsChains() { pkgSets := make([]rpmmd.PackageSet, len(setNames)) + + // add package-set-specific repositories to each set if one is defined for idx, pkgSetName := range setNames { pkgSets[idx] = packageSets[pkgSetName] - psRepos = append(psRepos, packageSetsRepos[pkgSetName]) // will be nil if it doesn't exist - delete(packageSets, pkgSetName) // will be depsolved here: remove from map + pkgSets[idx].Repositories = packageSetsRepos[pkgSetName] // will be nil if it doesn't exist + delete(packageSets, pkgSetName) // will be depsolved here: remove from map } - res, err := solver.Depsolve(pkgSets, imageTypeRepos, psRepos) + res, err := solver.Depsolve(pkgSets, imageTypeRepos) if err != nil { return nil, err } @@ -2170,7 +2166,8 @@ func (api *API) depsolveBlueprintForImageType(bp blueprint.Blueprint, imageType // depsolve the rest of the package sets for name, pkgSet := range packageSets { - res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, imageTypeRepos, [][]rpmmd.RepoConfig{packageSetsRepos[name]}) + pkgSet.Repositories = packageSetsRepos[name] + res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, imageTypeRepos) if err != nil { return nil, err } @@ -3209,7 +3206,7 @@ func (api *API) depsolveBlueprint(bp blueprint.Blueprint) ([]rpmmd.PackageSpec, } solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName) - solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: bp.GetPackages()}}, repos, nil) + solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: bp.GetPackages()}}, repos) if err != nil { return nil, err }