From 3481e1d3bac7ff00701556cbb2ba892bab58efe4 Mon Sep 17 00:00:00 2001 From: "Brian C. Lane" Date: Wed, 1 Mar 2023 15:06:33 -0800 Subject: [PATCH] Change the rpmmd cache directory structure to include the distro name This causes dnf-json to use separate caches, allowing them to run in parallel, with one lock per distribution. Multiple depsolves with the same distribution in the blueprint will continue to be serial. --- cmd/gen-manifests/main.go | 2 +- cmd/osbuild-dnf-json-tests/main_test.go | 4 +-- cmd/osbuild-pipeline/main.go | 2 +- cmd/osbuild-playground/playground.go | 2 +- cmd/osbuild-store-dump/main.go | 2 +- cmd/osbuild-worker/jobimpl-depsolve.go | 2 +- .../distro_test_common/distro_test_common.go | 6 +++- internal/dnfjson/dnfjson.go | 30 +++++++++++++++---- internal/dnfjson/dnfjson_test.go | 8 ++--- internal/weldr/api.go | 13 ++++---- 10 files changed, 47 insertions(+), 24 deletions(-) diff --git a/cmd/gen-manifests/main.go b/cmd/gen-manifests/main.go index c02946a7a..22ebc2508 100644 --- a/cmd/gen-manifests/main.go +++ b/cmd/gen-manifests/main.go @@ -245,7 +245,7 @@ func resolveContainers(containers []blueprint.Container, archName string) ([]con } func depsolve(cacheDir string, imageType distro.ImageType, bp blueprint.Blueprint, options distro.ImageOptions, repos []rpmmd.RepoConfig, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) { - solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, cacheDir) + solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) solver.SetDNFJSONPath("./dnf-json") packageSets := imageType.PackageSets(bp, options, repos) depsolvedSets := make(map[string][]rpmmd.PackageSpec) diff --git a/cmd/osbuild-dnf-json-tests/main_test.go b/cmd/osbuild-dnf-json-tests/main_test.go index 70c9427cb..9301bf291 100644 --- a/cmd/osbuild-dnf-json-tests/main_test.go +++ b/cmd/osbuild-dnf-json-tests/main_test.go @@ -38,7 +38,7 @@ func TestCrossArchDepsolve(t *testing.T) { t.Run(archStr, func(t *testing.T) { arch, err := cs9.GetArch(archStr) require.NoError(t, err) - solver := baseSolver.NewWithConfig(cs9.ModulePlatformID(), cs9.Releasever(), archStr) + solver := baseSolver.NewWithConfig(cs9.ModulePlatformID(), cs9.Releasever(), archStr, cs9.Name()) for _, imgTypeStr := range arch.ListImageTypes() { t.Run(imgTypeStr, func(t *testing.T) { imgType, err := arch.GetImageType(imgTypeStr) @@ -75,7 +75,7 @@ func TestDepsolvePackageSets(t *testing.T) { // Set up temporary directory for rpm/dnf cache dir := t.TempDir() - solver := dnfjson.NewSolver(cs9.ModulePlatformID(), cs9.Releasever(), distro.X86_64ArchName, dir) + solver := dnfjson.NewSolver(cs9.ModulePlatformID(), cs9.Releasever(), distro.X86_64ArchName, cs9.Name(), dir) repos, err := rpmmd.LoadRepositories([]string{repoDir}, cs9.Name()) require.NoErrorf(t, err, "Failed to LoadRepositories %v", cs9.Name()) diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go index 85bf98ccf..ed4c1e459 100644 --- a/cmd/osbuild-pipeline/main.go +++ b/cmd/osbuild-pipeline/main.go @@ -177,7 +177,7 @@ func main() { panic("os.UserHomeDir(): " + err.Error()) } - solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), path.Join(home, ".cache/osbuild-composer/rpmmd")) + solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), d.Name(), path.Join(home, ".cache/osbuild-composer/rpmmd")) solver.SetDNFJSONPath(findDnfJsonBin()) // Set cache size to 3 GiB diff --git a/cmd/osbuild-playground/playground.go b/cmd/osbuild-playground/playground.go index c280a437a..4687db083 100644 --- a/cmd/osbuild-playground/playground.go +++ b/cmd/osbuild-playground/playground.go @@ -17,7 +17,7 @@ import ( func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos map[string][]rpmmd.RepoConfig, state_dir string) { - solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), path.Join(state_dir, "rpmmd")) + solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), d.Name(), path.Join(state_dir, "rpmmd")) solver.SetDNFJSONPath(findDnfJsonBin()) // Set cache size to 3 GiB diff --git a/cmd/osbuild-store-dump/main.go b/cmd/osbuild-store-dump/main.go index 9ee01bd49..df71afb40 100644 --- a/cmd/osbuild-store-dump/main.go +++ b/cmd/osbuild-store-dump/main.go @@ -22,7 +22,7 @@ import ( func getManifest(bp blueprint.Blueprint, t distro.ImageType, a distro.Arch, d distro.Distro, cacheDir string, repos []rpmmd.RepoConfig) (distro.Manifest, []rpmmd.PackageSpec) { packageSets := t.PackageSets(bp, distro.ImageOptions{}, repos) pkgSpecSets := make(map[string][]rpmmd.PackageSpec) - solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), a.Name(), cacheDir) + solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), a.Name(), d.Name(), cacheDir) for name, packages := range packageSets { res, err := solver.Depsolve(packages) if err != nil { diff --git a/cmd/osbuild-worker/jobimpl-depsolve.go b/cmd/osbuild-worker/jobimpl-depsolve.go index 171deb88f..b5288f8f4 100644 --- a/cmd/osbuild-worker/jobimpl-depsolve.go +++ b/cmd/osbuild-worker/jobimpl-depsolve.go @@ -20,7 +20,7 @@ type DepsolveJobImpl struct { // packageSetsRepos are only used for the package set with the same name // (matching map keys). func (impl *DepsolveJobImpl) depsolve(packageSets map[string][]rpmmd.PackageSet, modulePlatformID, arch, releasever string) (map[string][]rpmmd.PackageSpec, error) { - solver := impl.Solver.NewWithConfig(modulePlatformID, releasever, arch) + solver := impl.Solver.NewWithConfig(modulePlatformID, releasever, arch, "") depsolvedSets := make(map[string][]rpmmd.PackageSpec) for name, pkgSet := range packageSets { diff --git a/internal/distro/distro_test_common/distro_test_common.go b/internal/distro/distro_test_common/distro_test_common.go index 8de5edbd0..b29e1279b 100644 --- a/internal/distro/distro_test_common/distro_test_common.go +++ b/internal/distro/distro_test_common/distro_test_common.go @@ -164,7 +164,11 @@ func TestDistro_Manifest(t *testing.T, pipelinePath string, prefix string, regis func getImageTypePkgSpecSets(imageType distro.ImageType, bp blueprint.Blueprint, options distro.ImageOptions, repos []rpmmd.RepoConfig, cacheDir, dnfJsonPath string) map[string][]rpmmd.PackageSpec { imgPackageSets := imageType.PackageSets(bp, options, repos) - solver := dnfjson.NewSolver(imageType.Arch().Distro().ModulePlatformID(), imageType.Arch().Distro().Releasever(), imageType.Arch().Name(), cacheDir) + solver := dnfjson.NewSolver(imageType.Arch().Distro().ModulePlatformID(), + imageType.Arch().Distro().Releasever(), + imageType.Arch().Name(), + imageType.Arch().Distro().Name(), + cacheDir) solver.SetDNFJSONPath(dnfJsonPath) depsolvedSets := make(map[string][]rpmmd.PackageSpec) for name, packages := range imgPackageSets { diff --git a/internal/dnfjson/dnfjson.go b/internal/dnfjson/dnfjson.go index 653274fcc..7587fca21 100644 --- a/internal/dnfjson/dnfjson.go +++ b/internal/dnfjson/dnfjson.go @@ -20,6 +20,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "sort" "strings" "time" @@ -68,12 +69,13 @@ func (s *BaseSolver) SetDNFJSONPath(cmd string, args ...string) { // NewWithConfig initialises a Solver with the platform information and the // BaseSolver's subscription info, cache directory, and dnf-json path. // Also loads system subscription information. -func (bs *BaseSolver) NewWithConfig(modulePlatformID string, releaseVer string, arch string) *Solver { +func (bs *BaseSolver) NewWithConfig(modulePlatformID, releaseVer, arch, distro string) *Solver { s := new(Solver) s.BaseSolver = *bs s.modulePlatformID = modulePlatformID s.arch = arch s.releaseVer = releaseVer + s.distro = distro subs, _ := rhsm.LoadSystemSubscriptions() s.subscriptions = subs return s @@ -102,13 +104,29 @@ type Solver struct { // system and required for subscription support. releaseVer string + // Full distribution string, eg. fedora-38, used to create separate dnf cache directories + // for each distribution. + distro string + subscriptions *rhsm.Subscriptions } // Create a new Solver with the given configuration. Initialising a Solver also loads system subscription information. -func NewSolver(modulePlatformID string, releaseVer string, arch string, cacheDir string) *Solver { +func NewSolver(modulePlatformID, releaseVer, arch, distro, cacheDir string) *Solver { s := NewBaseSolver(cacheDir) - return s.NewWithConfig(modulePlatformID, releaseVer, arch) + return s.NewWithConfig(modulePlatformID, releaseVer, arch, distro) +} + +// GetCacheDir returns a distro specific rpm cache directory +// It ensures that the distro name is below the root cache directory, and if there is +// a problem it returns the root cache intead of an error. +func (s *Solver) GetCacheDir() string { + b := filepath.Base(s.distro) + if b == "." || b == "/" { + return s.cache.root + } + + return filepath.Join(s.cache.root, s.distro) } // Depsolve the list of required package sets with explicit excludes using @@ -376,7 +394,7 @@ func (s *Solver) makeDepsolveRequest(pkgSets []rpmmd.PackageSet) (*Request, map[ Command: "depsolve", ModulePlatformID: s.modulePlatformID, Arch: s.arch, - CacheDir: s.cache.root, + CacheDir: s.GetCacheDir(), Arguments: args, } @@ -393,7 +411,7 @@ func (s *Solver) makeDumpRequest(repos []rpmmd.RepoConfig) (*Request, error) { Command: "dump", ModulePlatformID: s.modulePlatformID, Arch: s.arch, - CacheDir: s.cache.root, + CacheDir: s.GetCacheDir(), Arguments: arguments{ Repos: dnfRepos, }, @@ -411,7 +429,7 @@ func (s *Solver) makeSearchRequest(repos []rpmmd.RepoConfig, packages []string) Command: "search", ModulePlatformID: s.modulePlatformID, Arch: s.arch, - CacheDir: s.cache.root, + CacheDir: s.GetCacheDir(), Arguments: arguments{ Repos: dnfRepos, Search: searchArgs{ diff --git a/internal/dnfjson/dnfjson_test.go b/internal/dnfjson/dnfjson_test.go index 705f1c8c2..8ab8467a1 100644 --- a/internal/dnfjson/dnfjson_test.go +++ b/internal/dnfjson/dnfjson_test.go @@ -38,7 +38,7 @@ func TestDepsolver(t *testing.T) { assert := assert.New(t) tmpdir := t.TempDir() - solver := NewSolver("platform:el9", "9", "x86_64", tmpdir) + solver := NewSolver("platform:el9", "9", "x86_64", "rhel9.0", tmpdir) solver.SetDNFJSONPath("../../dnf-json") { // single depsolve @@ -346,7 +346,7 @@ func TestMakeDepsolveRequest(t *testing.T) { err: true, }, } - solver := NewSolver("", "", "", "") + solver := NewSolver("", "", "", "", "") for idx, tt := range tests { t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { req, _, err := solver.makeDepsolveRequest(tt.packageSets) @@ -532,7 +532,7 @@ func TestErrorRepoInfo(t *testing.T) { }, } - solver := NewSolver("f36", "36", "x86_64", "/tmp/cache") + solver := NewSolver("f36", "36", "x86_64", "fedora-36", "/tmp/cache") solver.SetDNFJSONPath("../../dnf-json") for idx, tc := range testCases { t.Run(fmt.Sprintf("%d", idx), func(t *testing.T) { @@ -565,7 +565,7 @@ func TestRepoConfigHash(t *testing.T) { } func TestRequestHash(t *testing.T) { - solver := NewSolver("f36", "36", "x86_64", "/tmp/cache") + solver := NewSolver("f36", "36", "x86_64", "fedora-36", "/tmp/cache") repos := []rpmmd.RepoConfig{ rpmmd.RepoConfig{ Name: "A test repository", diff --git a/internal/weldr/api.go b/internal/weldr/api.go index b7e2195dd..50425fcc3 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -321,7 +321,7 @@ func (api *API) PreloadMetadata() { return } - solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName) + solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name()) _, err = solver.Depsolve([]rpmmd.PackageSet{{Include: []string{"filesystem"}, Repositories: repos}}) if err != nil { log.Printf("Problem preloading distro metadata for %s: %s", distro, err) @@ -1267,7 +1267,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req return } - solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName) + solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name()) for i := range packageInfos { pkgName := packageInfos[i].Name solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: []string{pkgName}, Repositories: repos}}) @@ -1351,7 +1351,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt return } - solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName) + solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name()) deps, err := solver.Depsolve([]rpmmd.PackageSet{{Include: names, Repositories: repos}}) if err != nil { errors := responseError{ @@ -2228,7 +2228,8 @@ func (api *API) depsolveBlueprintForImageType(bp blueprint.Blueprint, options di } platformID := imageType.Arch().Distro().ModulePlatformID() releasever := imageType.Arch().Distro().Releasever() - solver := api.solver.NewWithConfig(platformID, releasever, api.archName) + distroName := imageType.Arch().Distro().Name() + solver := api.solver.NewWithConfig(platformID, releasever, api.archName, distroName) packageSets := imageType.PackageSets(bp, options, imageTypeRepos) depsolvedSets := make(map[string][]rpmmd.PackageSpec, len(packageSets)) @@ -3309,7 +3310,7 @@ func (api *API) fetchPackageList(distroName string, names []string) (packages rp return nil, err } - solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName) + solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name()) if len(names) == 0 { packages, err = solver.FetchMetadata(repos) } else { @@ -3387,7 +3388,7 @@ func (api *API) depsolveBlueprint(bp blueprint.Blueprint) ([]rpmmd.PackageSpec, return nil, err } - solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName) + solver := api.solver.NewWithConfig(d.ModulePlatformID(), d.Releasever(), api.archName, d.Name()) solved, err := solver.Depsolve([]rpmmd.PackageSet{{Include: bp.GetPackages(), Repositories: repos}}) if err != nil { return nil, err