particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Change the rpmmd cache directory structure to include the distro name

Browse source 
This causes dnf-json to use separate caches, allowing them to run in
parallel, with one lock per distribution. Multiple depsolves with the
same distribution in the blueprint will continue to be serial.
This commit is contained in:
Brian C. Lane 2023-03-01 15:06:33 -08:00 • committed by Achilleas Koutsou
parent 2ab0430d5d
commit 3481e1d3ba
10 changed files with 47 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/gen-manifests/main.go
View file

@ -245,7 +245,7 @@ func resolveContainers(containers []blueprint.Container, archName string) ([]con
}
func depsolve(cacheDir string, imageType distro.ImageType, bp blueprint.Blueprint, options distro.ImageOptions, repos []rpmmd.RepoConfig, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) {
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, cacheDir)
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir)
solver.SetDNFJSONPath("./dnf-json")
packageSets := imageType.PackageSets(bp, options, repos)
depsolvedSets := make(map[string][]rpmmd.PackageSpec)

4
cmd/osbuild-dnf-json-tests/main_test.go
View file

@ -38,7 +38,7 @@ func TestCrossArchDepsolve(t *testing.T) {
t.Run(archStr, func(t *testing.T) {
arch, err := cs9.GetArch(archStr)
require.NoError(t, err)
solver := baseSolver.NewWithConfig(cs9.ModulePlatformID(), cs9.Releasever(), archStr)
solver := baseSolver.NewWithConfig(cs9.ModulePlatformID(), cs9.Releasever(), archStr, cs9.Name())
for _, imgTypeStr := range arch.ListImageTypes() {
t.Run(imgTypeStr, func(t *testing.T) {
imgType, err := arch.GetImageType(imgTypeStr)
@ -75,7 +75,7 @@ func TestDepsolvePackageSets(t *testing.T) {
// Set up temporary directory for rpm/dnf cache
dir := t.TempDir()
solver := dnfjson.NewSolver(cs9.ModulePlatformID(), cs9.Releasever(), distro.X86_64ArchName, dir)
solver := dnfjson.NewSolver(cs9.ModulePlatformID(), cs9.Releasever(), distro.X86_64ArchName, cs9.Name(), dir)
repos, err := rpmmd.LoadRepositories([]string{repoDir}, cs9.Name())
require.NoErrorf(t, err, "Failed to LoadRepositories %v", cs9.Name())

2
cmd/osbuild-pipeline/main.go
View file

@ -177,7 +177,7 @@ func main() {
panic("os.UserHomeDir(): " + err.Error())
}
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), path.Join(home, ".cache/osbuild-composer/rpmmd"))
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), d.Name(), path.Join(home, ".cache/osbuild-composer/rpmmd"))
solver.SetDNFJSONPath(findDnfJsonBin())
// Set cache size to 3 GiB

2
cmd/osbuild-playground/playground.go
View file

@ -17,7 +17,7 @@ import (
func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos map[string][]rpmmd.RepoConfig, state_dir string) {
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), path.Join(state_dir, "rpmmd"))
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), d.Name(), path.Join(state_dir, "rpmmd"))
solver.SetDNFJSONPath(findDnfJsonBin())
// Set cache size to 3 GiB

2
cmd/osbuild-store-dump/main.go
View file

@ -22,7 +22,7 @@ import (
func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d distro.Distro, cacheDir string, repos []rpmmd.RepoConfig) (distro.Manifest, []rpmmd.PackageSpec) {
packageSets := t.PackageSets(bp, distro.ImageOptions{}, repos)
pkgSpecSets := make(map[string][]rpmmd.PackageSpec)
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), a.Name(), cacheDir)
solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), a.Name(), d.Name(), cacheDir)
for name, packages := range packageSets {
res, err := solver.Depsolve(packages)
if err != nil {

2
cmd/osbuild-worker/jobimpl-depsolve.go
View file

@ -20,7 +20,7 @@ type DepsolveJobImpl struct {
// packageSetsRepos are only used for the package set with the same name
// (matching map keys).
func (impl *DepsolveJobImpl) depsolve(packageSets map[string][]rpmmd.PackageSet, modulePlatformID, arch, releasever string) (map[string][]rpmmd.PackageSpec, error) {
solver := impl.Solver.NewWithConfig(modulePlatformID, releasever, arch)
solver := impl.Solver.NewWithConfig(modulePlatformID, releasever, arch, "")
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
for name, pkgSet := range packageSets {

6
internal/distro/distro_test_common/distro_test_common.go
View file

@ -164,7 +164,11 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis
func getImageTypePkgSpecSets(imageType distro.ImageType, bp blueprint.Blueprint, options distro.ImageOptions, repos []rpmmd.RepoConfig, cacheDir, dnfJsonPath string) map[string][]rpmmd.PackageSpec {
imgPackageSets := imageType.PackageSets(bp, options, repos)
solver := dnfjson.NewSolver(imageType.Arch().Distro().ModulePlatformID(), imageType.Arch().Distro().Releasever(), imageType.Arch().Name(), cacheDir)
solver := dnfjson.NewSolver(imageType.Arch().Distro().ModulePlatformID(),
imageType.Arch().Distro().Releasever(),
imageType.Arch().Name(),
imageType.Arch().Distro().Name(),
cacheDir)
solver.SetDNFJSONPath(dnfJsonPath)
depsolvedSets := make(map[string][]rpmmd.PackageSpec)
for name, packages := range imgPackageSets {

30
internal/dnfjson/dnfjson.go
View file

@ -20,6 +20,7 @@ import (
"fmt"
"os"
"os/exec"
"path/filepath"
"sort"
"strings"
"time"
@ -68,12 +69,13 @@ func (s *BaseSolver) SetDNFJSONPath(cmd string, args ...string) {
// NewWithConfig initialises a Solver with the platform information and the
// BaseSolver's subscription info, cache directory, and dnf-json path.
// Also loads system subscription information.
func (bs *BaseSolver) NewWithConfig(modulePlatformID string, releaseVer string, arch string) *Solver {
func (bs *BaseSolver) NewWithConfig(modulePlatformID, releaseVer, arch, distro string) *Solver {
s := new(Solver)
s.BaseSolver = *bs
s.modulePlatformID = modulePlatformID
s.arch = arch
s.releaseVer = releaseVer
s.distro = distro
subs, _ := rhsm.LoadSystemSubscriptions()
s.subscriptions = subs
return s
@ -102,13 +104,29 @@ type Solver struct {
// system and required for subscription support.
releaseVer string
// Full distribution string, eg. fedora-38, used to create separate dnf cache directories
// for each distribution.
distro string
subscriptions *rhsm.Subscriptions
}
// Create a new Solver with the given configuration. Initialising a Solver also loads system subscription information.
func NewSolver(modulePlatformID string, releaseVer string, arch string, cacheDir string) *Solver {
func NewSolver(modulePlatformID, releaseVer, arch, distro, cacheDir string) *Solver {
s := NewBaseSolver(cacheDir)
return s.NewWithConfig(modulePlatformID, releaseVer, arch)
return s.NewWithConfig(modulePlatformID, releaseVer, arch, distro)
}
// GetCacheDir returns a distro specific rpm cache directory
// It ensures that the distro name is below the root cache directory, and if there is
// a problem it returns the root cache intead of an error.
func (s *Solver) GetCacheDir() string {
b := filepath.Base(s.distro)
if b == "." || b == "/" {
return s.cache.root
}
return filepath.Join(s.cache.root, s.distro)
}
// Depsolve the list of required package sets with explicit excludes using
@ -376,7 +394,7 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[
Command: "depsolve",
ModulePlatformID: s.modulePlatformID,
Arch: s.arch,
CacheDir: s.cache.root,
CacheDir: s.GetCacheDir(),
Arguments: args,
}
@ -393,7 +411,7 @@ func (s *Solver) makeDumpRequest(repos []rpmmd.RepoConfig) (*Request, error) {
Command: "dump",
ModulePlatformID: s.modulePlatformID,
Arch: s.arch,
CacheDir: s.cache.root,
CacheDir: s.GetCacheDir(),
Arguments: arguments{
Repos: dnfRepos,
},
@ -411,7 +429,7 @@ func (s *Solver) makeSearchRequest(repos []rpmmd.RepoConfig, packages []string)
Command: "search",
ModulePlatformID: s.modulePlatformID,
Arch: s.arch,
CacheDir: s.cache.root,
CacheDir: s.GetCacheDir(),
Arguments: arguments{
Repos: dnfRepos,
Search: searchArgs{

8
internal/dnfjson/dnfjson_test.go
View file

@ -38,7 +38,7 @@ func TestDepsolver(t *testing.T) {
assert := assert.New(t)
tmpdir := t.TempDir()
solver := NewSolver("platform:el9", "9", "x86_64", tmpdir)
solver := NewSolver("platform:el9", "9", "x86_64", "rhel9.0", tmpdir)
solver.SetDNFJSONPath("../../dnf-json")
{ // single depsolve
@ -346,7 +346,7 @@ func TestMakeDepsolveRequest(t *testing.T) {
err: true,
},
}
solver := NewSolver("", "", "", "")
solver := NewSolver("", "", "", "", "")
for idx, tt := range tests {
t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
req, _, err := solver.makeDepsolveRequest(tt.packageSets)
@ -532,7 +532,7 @@ func TestErrorRepoInfo(t *testing.T) {
},
}
solver := NewSolver("f36", "36", "x86_64", "/tmp/cache")
solver := NewSolver("f36", "36", "x86_64", "fedora-36", "/tmp/cache")
solver.SetDNFJSONPath("../../dnf-json")
for idx, tc := range testCases {
t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) {
@ -565,7 +565,7 @@ func TestRepoConfigHash(t *testing.T) {
}
func TestRequestHash(t *testing.T) {
solver := NewSolver("f36", "36", "x86_64", "/tmp/cache")
solver := NewSolver("f36", "36", "x86_64", "fedora-36", "/tmp/cache")
repos := []rpmmd.RepoConfig{
rpmmd.RepoConfig{
Name: "A test repository",

13
internal/weldr/api.go
View file

@ -321,7 +321,7 @@ func (api *API) PreloadMetadata() {
return
}
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName)
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name())
_, err = solver.Depsolve([]rpmmd.PackageSet{{Include: []string{"filesystem"}, Repositories: repos}})
if err != nil {
log.Printf("Problem preloading distro metadata for %s: %s", distro, err)
@ -1267,7 +1267,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req
return
}
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName)
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name())
for i := range packageInfos {
pkgName := packageInfos[i].Name
solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: []string{pkgName}, Repositories: repos}})
@ -1351,7 +1351,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt
return
}
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName)
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name())
deps, err := solver.Depsolve([]rpmmd.PackageSet{{Include: names, Repositories: repos}})
if err != nil {
errors := responseError{
@ -2228,7 +2228,8 @@ func (api *API) depsolveBlueprintForImageType(bp blueprint.Blueprint, options di
}
platformID := imageType.Arch().Distro().ModulePlatformID()
releasever := imageType.Arch().Distro().Releasever()
solver := api.solver.NewWithConfig(platformID, releasever, api.archName)
distroName := imageType.Arch().Distro().Name()
solver := api.solver.NewWithConfig(platformID, releasever, api.archName, distroName)
packageSets := imageType.PackageSets(bp, options, imageTypeRepos)
depsolvedSets := make(map[string][]rpmmd.PackageSpec, len(packageSets))
@ -3309,7 +3310,7 @@ func (api *API) fetchPackageList(distroName string, names []string) (packages rp
return nil, err
}
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName)
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name())
if len(names) == 0 {
packages, err = solver.FetchMetadata(repos)
} else {
@ -3387,7 +3388,7 @@ func (api *API) depsolveBlueprint(bp blueprint.Blueprint) ([]rpmmd.PackageSpec,
return nil, err
}
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName)
solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name())
solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: bp.GetPackages(), Repositories: repos}})
if err != nil {
return nil, err