particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rpmmd: add Repositories list to PackageSet struct

Browse source 
Attach the repository configurations that are specific to a package set
directly on the PackageSet object.  This simplifies the Depsolve()
signature and avoids requiring a `nil` when no additional repositories
are required.  More importantly, it makes associating repositories to
package sets explicit, no longer relying on matching array indices or
map keys.
This commit is contained in:
Achilleas Koutsou 2022-05-05 11:56:39 +02:00 • committed by Tom Gundersen
parent 1c4d8f9988
commit 86536f11e7
10 changed files with 72 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

8
cmd/osbuild-dnf-json-tests/main_test.go
View file

@ -83,10 +83,10 @@ func TestCrossArchDepsolve(t *testing.T) {
packages := imgType.PackageSets(blueprint.Blueprint{})
_, err = solver.Depsolve([]rpmmd.PackageSet{packages["build"]}, repos[archStr], nil)
_, err = solver.Depsolve([]rpmmd.PackageSet{packages["build"]}, repos[archStr])
assert.NoError(t, err)
_, err = solver.Depsolve([]rpmmd.PackageSet{packages["packages"]}, repos[archStr], nil)
_, err = solver.Depsolve([]rpmmd.PackageSet{packages["packages"]}, repos[archStr])
assert.NoError(t, err)
})
}
@ -155,7 +155,7 @@ func TestDepsolvePackageSets(t *testing.T) {
pkgSets[idx] = imagePkgSets[pkgSetName]
delete(imagePkgSets, pkgSetName) // will be depsolved here: remove from map
}
res, err := solver.Depsolve(pkgSets, x86Repos, nil)
res, err := solver.Depsolve(pkgSets, x86Repos)
if err != nil {
require.Nil(t, err)
}
@ -164,7 +164,7 @@ func TestDepsolvePackageSets(t *testing.T) {
// depsolve the rest of the package sets
for name, pkgSet := range imagePkgSets {
res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, x86Repos, nil)
res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, x86Repos)
if err != nil {
require.Nil(t, err)
}

4
cmd/osbuild-pipeline/main.go
View file

@ -152,7 +152,7 @@ func main() {
pkgSets[idx] = packageSets[pkgSetName]
delete(packageSets, pkgSetName) // will be depsolved here: remove from map
}
res, err := solver.Depsolve(pkgSets, repos, nil)
res, err := solver.Depsolve(pkgSets, repos)
if err != nil {
panic("Could not depsolve: " + err.Error())
}
@ -161,7 +161,7 @@ func main() {
// depsolve the rest of the package sets
for name, pkgSet := range packageSets {
res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, repos, nil)
res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, repos)
if err != nil {
panic("Could not depsolve: " + err.Error())
}

2
cmd/osbuild-store-dump/main.go
View file

@ -23,7 +23,7 @@ func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d di
pkgSpecSets := make(map[string][]rpmmd.PackageSpec)
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), a.Name(), cacheDir)
for name, packages := range packageSets {
res, err := solver.Depsolve([]rpmmd.PackageSet{packages}, repos, nil)
res, err := solver.Depsolve([]rpmmd.PackageSet{packages}, repos)
if err != nil {
panic(err)
}

12
cmd/osbuild-worker/jobimpl-depsolve.go
View file

@ -21,18 +21,17 @@ type DepsolveJobImpl struct {
// (matching map keys).
func (impl *DepsolveJobImpl) depsolve(packageSetsChains map[string][]string, packageSets map[string]rpmmd.PackageSet, repos []rpmmd.RepoConfig, packageSetsRepos map[string][]rpmmd.RepoConfig, modulePlatformID, arch, releasever string) (map[string][]rpmmd.PackageSpec, error) {
solver := dnfjson.NewSolver(modulePlatformID, releasever, arch, impl.RPMMDCache)
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
psRepos := make([][]rpmmd.RepoConfig, 0)
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
// first depsolve package sets that are part of a chain
for specName, setNames := range packageSetsChains {
pkgSets := make([]rpmmd.PackageSet, len(setNames))
for idx, pkgSetName := range setNames {
pkgSets[idx] = packageSets[pkgSetName]
psRepos = append(psRepos, packageSetsRepos[pkgSetName]) // will be nil if it doesn't exist
delete(packageSets, pkgSetName) // will be depsolved here: remove from map
pkgSets[idx].Repositories = packageSetsRepos[pkgSetName]
delete(packageSets, pkgSetName) // will be depsolved here: remove from map
}
res, err := solver.Depsolve(pkgSets, repos, psRepos)
res, err := solver.Depsolve(pkgSets, repos)
if err != nil {
return nil, err
}
@ -41,7 +40,8 @@ func (impl *DepsolveJobImpl) depsolve(packageSetsChains map[string][]string, pac
// depsolve the rest of the package sets
for name, pkgSet := range packageSets {
res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, repos, [][]rpmmd.RepoConfig{packageSetsRepos[name]})
pkgSet.Repositories = packageSetsRepos[name]
res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, repos)
if err != nil {
return nil, err
}

2
internal/distro/distro_test_common/distro_test_common.go
View file

@ -143,7 +143,7 @@ func getImageTypePkgSpecSets(imageType distro.ImageType, bp blueprint.Blueprint,
solver := dnfjson.NewSolver(imageType.Arch().Distro().ModulePlatformID(), imageType.Arch().Distro().Releasever(), imageType.Arch().Name(), cacheDir)
imgPackageSpecSets := make(map[string][]rpmmd.PackageSpec)
for name, packages := range imgPackageSets {
res, err := solver.Depsolve([]rpmmd.PackageSet{packages}, repos, nil)
res, err := solver.Depsolve([]rpmmd.PackageSet{packages}, repos)
if err != nil {
panic("Could not depsolve: " + err.Error())
}

31
internal/dnfjson/dnfjson.go
View file

@ -87,16 +87,16 @@ func NewSolver(modulePlatformID string, releaseVer string, arch string, cacheDir
}
// Depsolve the given packages with explicit excludes using the given configuration and repos
func Depsolve(pkgSets []rpmmd.PackageSet, repos []rpmmd.RepoConfig, psRepos [][]rpmmd.RepoConfig, modulePlatformID string, releaseVer string, arch string, cacheDir string) (*DepsolveResult, error) {
return NewSolver(modulePlatformID, releaseVer, arch, cacheDir).Depsolve(pkgSets, repos, psRepos)
func Depsolve(pkgSets []rpmmd.PackageSet, repos []rpmmd.RepoConfig, modulePlatformID string, releaseVer string, arch string, cacheDir string) (*DepsolveResult, error) {
return NewSolver(modulePlatformID, releaseVer, arch, cacheDir).Depsolve(pkgSets, repos)
}
// Depsolve the list of required package sets with explicit excludes using
// the given repositories. Each package set is depsolved as a separate
// transactions in a chain. It returns a list of all packages (with solved
// dependencies) that will be installed into the system.
func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet, repos []rpmmd.RepoConfig, psRepos [][]rpmmd.RepoConfig) (*DepsolveResult, error) {
req, repoMap, err := s.makeDepsolveRequest(pkgSets, repos, psRepos)
func (s *Solver) Depsolve(pkgSets []rpmmd.PackageSet, baseRepos []rpmmd.RepoConfig) (*DepsolveResult, error) {
req, repoMap, err := s.makeDepsolveRequest(pkgSets, baseRepos)
if err != nil {
return nil, err
}
@ -205,11 +205,7 @@ type repoConfig struct {
// NOTE: Due to implementation limitations of DNF and dnf-json, each package set
// in the chain must use all of the repositories used by its predecessor.
// An error is returned if this requirement is not met.
func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet, repoConfigs []rpmmd.RepoConfig, pkgsetsRepos [][]rpmmd.RepoConfig) (*Request, map[string]rpmmd.RepoConfig, error) {
// pkgsetsRepos must either be nil (empty) or the same length as the pkgSets array
if len(pkgsetsRepos) > 0 && len(pkgSets) != len(pkgsetsRepos) {
return nil, nil, fmt.Errorf("depsolve: the number of package set repository configurations (%d) does not match the number of package sets (%d)", len(pkgsetsRepos), len(pkgSets))
}
func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet, baseRepos []rpmmd.RepoConfig) (*Request, map[string]rpmmd.RepoConfig, error) {
// dedupe repository configurations but maintain order
// the order in which repositories are added to the request affects the
@ -218,17 +214,15 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet, repoConfigs []r
rpmRepoMap := make(map[string]rpmmd.RepoConfig)
// These repo IDs will be used for all transactions in the chain
baseRepoIDs := make([]string, len(repoConfigs))
for idx, repo := range repoConfigs {
baseRepoIDs := make([]string, len(baseRepos))
for idx, repo := range baseRepos {
id := repo.Hash()
rpmRepoMap[id] = repo
baseRepoIDs[idx] = id
repos = append(repos, repo)
}
// extra repositories defined for specific package sets
for _, jobRepos := range pkgsetsRepos {
for _, repo := range jobRepos {
for _, ps := range pkgSets {
for _, repo := range ps.Repositories {
id := repo.Hash()
if _, ok := rpmRepoMap[id]; !ok {
rpmRepoMap[id] = repo
@ -245,12 +239,7 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet, repoConfigs []r
RepoIDs: baseRepoIDs, // due to its capacity, the slice will be copied when appended to
}
if len(pkgsetsRepos) == 0 {
// nothing to do
continue
}
for _, jobRepo := range pkgsetsRepos[dsIdx] {
for _, jobRepo := range pkgSet.Repositories {
transactions[dsIdx].RepoIDs = append(transactions[dsIdx].RepoIDs, jobRepo.Hash())
}

54
internal/dnfjson/dnfjson_test.go
View file

@ -22,7 +22,7 @@ func TestDepsolver(t *testing.T) {
{ // single depsolve
pkgsets := []rpmmd.PackageSet{{Include: []string{"kernel", "vim-minimal", "tmux", "zsh"}}} // everything you'll ever need
deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig}, nil)
deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig})
if err != nil {
t.Fatal(err)
}
@ -35,7 +35,7 @@ func TestDepsolver(t *testing.T) {
{Include: []string{"kernel"}},
{Include: []string{"vim-minimal", "tmux", "zsh"}},
}
deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig}, nil)
deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig})
if err != nil {
t.Fatal(err)
}
@ -48,7 +48,8 @@ func TestDepsolver(t *testing.T) {
{Include: []string{"kernel"}},
{Include: []string{"vim-minimal", "tmux", "zsh"}},
}
deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig}, [][]rpmmd.RepoConfig{nil, {s.RepoConfig}})
pkgsets[1].Repositories = []rpmmd.RepoConfig{s.RepoConfig}
deps, err := solver.Depsolve(pkgsets, []rpmmd.RepoConfig{s.RepoConfig})
if err != nil {
t.Fatal(err)
}
@ -123,11 +124,11 @@ func TestMakeDepsolveRequest(t *testing.T) {
Exclude: []string{"pkg2"},
},
{
Include: []string{"pkg3"},
Include: []string{"pkg3"},
Repositories: []rpmmd.RepoConfig{userRepo},
},
},
repos: []rpmmd.RepoConfig{baseOS, appstream},
packageSetsRepos: [][]rpmmd.RepoConfig{nil, {userRepo}},
repos: []rpmmd.RepoConfig{baseOS, appstream},
args: []transactionArgs{
{
PackageSpecs: []string{"pkg1"},
@ -201,14 +202,15 @@ func TestMakeDepsolveRequest(t *testing.T) {
Exclude: []string{"pkg2"},
},
{
Include: []string{"pkg3"},
Include: []string{"pkg3"},
Repositories: []rpmmd.RepoConfig{userRepo},
},
{
Include: []string{"pkg4"},
Include: []string{"pkg4"},
Repositories: []rpmmd.RepoConfig{userRepo},
},
},
repos: []rpmmd.RepoConfig{baseOS, appstream},
packageSetsRepos: [][]rpmmd.RepoConfig{nil, {userRepo}, {userRepo}},
repos: []rpmmd.RepoConfig{baseOS, appstream},
args: []transactionArgs{
{
PackageSpecs: []string{"pkg1"},
@ -251,14 +253,15 @@ func TestMakeDepsolveRequest(t *testing.T) {
Exclude: []string{"pkg2"},
},
{
Include: []string{"pkg3"},
Include: []string{"pkg3"},
Repositories: []rpmmd.RepoConfig{userRepo},
},
{
Include: []string{"pkg4"},
Include: []string{"pkg4"},
Repositories: []rpmmd.RepoConfig{userRepo, userRepo2},
},
},
repos: []rpmmd.RepoConfig{baseOS, appstream},
packageSetsRepos: [][]rpmmd.RepoConfig{nil, {userRepo}, {userRepo, userRepo2}},
repos: []rpmmd.RepoConfig{baseOS, appstream},
args: []transactionArgs{
{
PackageSpecs: []string{"pkg1"},
@ -305,17 +308,18 @@ func TestMakeDepsolveRequest(t *testing.T) {
Exclude: []string{"pkg2"},
},
{
Include: []string{"pkg3"},
Include: []string{"pkg3"},
Repositories: []rpmmd.RepoConfig{userRepo},
},
{
Include: []string{"pkg4"},
Include: []string{"pkg4"},
Repositories: []rpmmd.RepoConfig{userRepo2},
},
},
repos: []rpmmd.RepoConfig{baseOS, appstream},
packageSetsRepos: [][]rpmmd.RepoConfig{nil, {userRepo}, {userRepo2}},
err: true,
repos: []rpmmd.RepoConfig{baseOS, appstream},
err: true,
},
// Error: 3 transactions but only 1 packageSetsRepos
// Error: 3 transactions but last one doesn't specify user repos in 2nd
{
packageSets: []rpmmd.PackageSet{
{
@ -323,21 +327,21 @@ func TestMakeDepsolveRequest(t *testing.T) {
Exclude: []string{"pkg2"},
},
{
Include: []string{"pkg3"},
Include: []string{"pkg3"},
Repositories: []rpmmd.RepoConfig{userRepo, userRepo2},
},
{
Include: []string{"pkg4"},
},
},
repos: []rpmmd.RepoConfig{baseOS, appstream},
packageSetsRepos: [][]rpmmd.RepoConfig{{userRepo, userRepo2}},
err: true,
repos: []rpmmd.RepoConfig{baseOS, appstream},
err: true,
},
}
solver := NewSolver("", "", "", "")
for idx, tt := range tests {
t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
req, _, err := solver.makeDepsolveRequest(tt.packageSets, tt.repos, tt.packageSetsRepos)
req, _, err := solver.makeDepsolveRequest(tt.packageSets, tt.repos)
if tt.err {
assert.NotNilf(t, err, "expected an error, but got 'nil' instead")
assert.Nilf(t, req, "got non-nill request, but expected an error")

2
internal/kojiapi/server.go
View file

@ -131,7 +131,7 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error {
for idx, pkgSetName := range setNames {
chain[idx] = packageSets[pkgSetName]
}
res, err := solver.Depsolve(chain, repositories, nil)
res, err := solver.Depsolve(chain, repositories)
if err != nil {
return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Failed to depsolve base packages for %s/%s/%s: %s", ir.ImageType, ir.Architecture, request.Distribution, err))
}

10
internal/rpmmd/repository.go
View file

@ -97,11 +97,13 @@ func (pkg Package) ToPackageInfo() PackageInfo {
}
}
// The inputs to depsolve, a set of packages to include and a set of
// packages to exclude.
// The inputs to depsolve, a set of packages to include and a set of packages
// to exclude. The Repositories are used when depsolving this package set in
// addition to the base repositories.
type PackageSet struct {
Include []string
Exclude []string
Include []string
Exclude []string
Repositories []RepoConfig
}
// Append the Include and Exclude package list from another PackageSet and

27
internal/weldr/api.go
View file

@ -1261,7 +1261,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName)
for i := range packageInfos {
pkgName := packageInfos[i].Name
solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: []string{pkgName}}}, repos, nil)
solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: []string{pkgName}}}, repos)
if err != nil {
errors := responseError{
ID: errorId,
@ -1339,11 +1339,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt
}
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName)
deps, err := solver.Depsolve(
[]rpmmd.PackageSet{{Include: names}},
repos,
nil,
)
deps, err := solver.Depsolve([]rpmmd.PackageSet{{Include: names}}, repos)
if err != nil {
errors := responseError{
ID: "ProjectsError",
@ -2145,23 +2141,23 @@ func (api *API) depsolveBlueprintForImageType(bp blueprint.Blueprint, imageType
packageSetsRepos[name] = payloadRepos
}
packageSets := imageType.PackageSets(bp)
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
platformID := imageType.Arch().Distro().ModulePlatformID()
releasever := imageType.Arch().Distro().Releasever()
solver := api.solver.NewWithConfig(platformID, releasever, api.archName)
psRepos := make([][]rpmmd.RepoConfig, 0)
packageSets := imageType.PackageSets(bp)
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
// first depsolve package sets that are part of a chain
for specName, setNames := range imageType.PackageSetsChains() {
pkgSets := make([]rpmmd.PackageSet, len(setNames))
// add package-set-specific repositories to each set if one is defined
for idx, pkgSetName := range setNames {
pkgSets[idx] = packageSets[pkgSetName]
psRepos = append(psRepos, packageSetsRepos[pkgSetName]) // will be nil if it doesn't exist
delete(packageSets, pkgSetName) // will be depsolved here: remove from map
pkgSets[idx].Repositories = packageSetsRepos[pkgSetName] // will be nil if it doesn't exist
delete(packageSets, pkgSetName) // will be depsolved here: remove from map
}
res, err := solver.Depsolve(pkgSets, imageTypeRepos, psRepos)
res, err := solver.Depsolve(pkgSets, imageTypeRepos)
if err != nil {
return nil, err
}
@ -2170,7 +2166,8 @@ func (api *API) depsolveBlueprintForImageType(bp blueprint.Blueprint, imageType
// depsolve the rest of the package sets
for name, pkgSet := range packageSets {
res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, imageTypeRepos, [][]rpmmd.RepoConfig{packageSetsRepos[name]})
pkgSet.Repositories = packageSetsRepos[name]
res, err := solver.Depsolve([]rpmmd.PackageSet{pkgSet}, imageTypeRepos)
if err != nil {
return nil, err
}
@ -3209,7 +3206,7 @@ func (api *API) depsolveBlueprint(bp blueprint.Blueprint) ([]rpmmd.PackageSpec,
}
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName)
solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: bp.GetPackages()}}, repos, nil)
solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: bp.GetPackages()}}, repos)
if err != nil {
return nil, err
}