From 819430e659b208a2bbe5cee4a897d4f0b3e7254b Mon Sep 17 00:00:00 2001 From: Tom Gundersen Date: Sat, 14 Mar 2020 20:47:15 +0100 Subject: [PATCH] rpmmd: no longer flush the caches on every call When we used the dnf-based pipelines, we were relying on the fact that the metadata was unlikely to have changed between we generated the pipeline and called osbuild. We achieved this by always updating to the most recent metadata on every call to rpmmd.Depsolve that would end up in a pipelin. Refreshing the metadata is time-consuming, and something we want to avoid if at all possible. Now that our pipelines no longer rely on this property, we can drop the flushing. Signed-off-by: Tom Gundersen --- cmd/osbuild-pipeline/main.go | 4 ++-- dnf-json | 8 ++------ internal/mocks/rpmmd/rpmmd_mock.go | 2 +- internal/rpmmd/repository.go | 9 ++++----- internal/weldr/api.go | 6 +++--- 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go index fbdf655d0..87904d9ae 100644 --- a/cmd/osbuild-pipeline/main.go +++ b/cmd/osbuild-pipeline/main.go @@ -112,7 +112,7 @@ func main() { } rpmmd := rpmmd.NewRPMMD(path.Join(home, ".cache/osbuild-composer/rpmmd")) - packageSpecs, checksums, err := rpmmd.Depsolve(packages, exclude_pkgs, d.Repositories(archArg), d.ModulePlatformID(), false) + packageSpecs, checksums, err := rpmmd.Depsolve(packages, exclude_pkgs, d.Repositories(archArg), d.ModulePlatformID()) if err != nil { panic("Could not depsolve: " + err.Error()) } @@ -121,7 +121,7 @@ func main() { if err != nil { panic("Could not get build packages: " + err.Error()) } - buildPackageSpecs, _, err := rpmmd.Depsolve(buildPkgs, nil, d.Repositories(archArg), d.ModulePlatformID(), false) + buildPackageSpecs, _, err := rpmmd.Depsolve(buildPkgs, nil, d.Repositories(archArg), d.ModulePlatformID()) if err != nil { panic("Could not depsolve build packages: " + err.Error()) } diff --git a/dnf-json b/dnf-json index a2909659f..bc4da2faf 100755 --- a/dnf-json +++ b/dnf-json @@ -38,16 +38,13 @@ def dnfrepo(desc, parent_conf=None): return repo -def create_base(repos, module_platform_id, persistdir, cachedir, clean=False): +def create_base(repos, module_platform_id, persistdir, cachedir): base = dnf.Base() base.conf.module_platform_id = module_platform_id base.conf.config_file_path = "/dev/null" base.conf.persistdir = persistdir base.conf.cachedir = cachedir - if clean: - shutil.rmtree(base.conf.cachedir, ignore_errors=True) - for repo in repos: base.repos.add(dnfrepo(repo, base.conf)) @@ -88,12 +85,11 @@ call = json.load(sys.stdin) command = call["command"] arguments = call["arguments"] repos = arguments.get("repos", {}) -clean = arguments.get("clean", False) cachedir = arguments["cachedir"] module_platform_id = arguments["module_platform_id"] with tempfile.TemporaryDirectory() as persistdir: - base = create_base(repos, module_platform_id, persistdir, cachedir, clean) + base = create_base(repos, module_platform_id, persistdir, cachedir) if command == "dump": packages = [] diff --git a/internal/mocks/rpmmd/rpmmd_mock.go b/internal/mocks/rpmmd/rpmmd_mock.go index fabbe7fda..35a7689ae 100644 --- a/internal/mocks/rpmmd/rpmmd_mock.go +++ b/internal/mocks/rpmmd/rpmmd_mock.go @@ -34,6 +34,6 @@ func (r *rpmmdMock) FetchMetadata(repos []rpmmd.RepoConfig, modulePlatformID str return r.Fixture.fetchPackageList.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.fetchPackageList.err } -func (r *rpmmdMock) Depsolve(specs, excludeSpecs []string, repos []rpmmd.RepoConfig, modulePlatformID string, clean bool) ([]rpmmd.PackageSpec, map[string]string, error) { +func (r *rpmmdMock) Depsolve(specs, excludeSpecs []string, repos []rpmmd.RepoConfig, modulePlatformID string) ([]rpmmd.PackageSpec, map[string]string, error) { return r.Fixture.depsolve.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.depsolve.err } diff --git a/internal/rpmmd/repository.go b/internal/rpmmd/repository.go index baff1b53c..46796dafb 100644 --- a/internal/rpmmd/repository.go +++ b/internal/rpmmd/repository.go @@ -102,7 +102,7 @@ type RPMMD interface { // Depsolve takes a list of required content (specs), explicitly unwanted content (excludeSpecs), list // or repositories, and platform ID for modularity. It returns a list of all packages (with solved // dependencies) that will be installed into the system. - Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID string, clean bool) ([]PackageSpec, map[string]string, error) + Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID string) ([]PackageSpec, map[string]string, error) } type DNFError struct { @@ -237,15 +237,14 @@ func (r *rpmmdImpl) FetchMetadata(repos []RepoConfig, modulePlatformID string) ( return reply.Packages, reply.Checksums, err } -func (r *rpmmdImpl) Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID string, clean bool) ([]PackageSpec, map[string]string, error) { +func (r *rpmmdImpl) Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID string) ([]PackageSpec, map[string]string, error) { var arguments = struct { PackageSpecs []string `json:"package-specs"` ExcludSpecs []string `json:"exclude-specs"` Repos []RepoConfig `json:"repos"` CacheDir string `json:"cachedir"` ModulePlatformID string `json:"module_platform_id"` - Clean bool `json:"clean,omitempty"` - }{specs, excludeSpecs, repos, r.CacheDir, modulePlatformID, clean} + }{specs, excludeSpecs, repos, r.CacheDir, modulePlatformID} var reply struct { Checksums map[string]string `json:"checksums"` Dependencies []PackageSpec `json:"dependencies"` @@ -307,6 +306,6 @@ func (packages PackageList) ToPackageInfos() []PackageInfo { } func (pkg *PackageInfo) FillDependencies(rpmmd RPMMD, repos []RepoConfig, modulePlatformID string) (err error) { - pkg.Dependencies, _, err = rpmmd.Depsolve([]string{pkg.Name}, nil, repos, modulePlatformID, false) + pkg.Dependencies, _, err = rpmmd.Depsolve([]string{pkg.Name}, nil, repos, modulePlatformID) return } diff --git a/internal/weldr/api.go b/internal/weldr/api.go index bfec7f8ff..0987417a7 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -653,7 +653,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt names := strings.Split(params.ByName("projects"), ",") - packages, _, err := api.rpmmd.Depsolve(names, nil, api.distro.Repositories(api.arch), api.distro.ModulePlatformID(), false) + packages, _, err := api.rpmmd.Depsolve(names, nil, api.distro.Repositories(api.arch), api.distro.ModulePlatformID()) if err != nil { errors := responseError{ @@ -2001,7 +2001,7 @@ func (api *API) depsolveBlueprint(bp *blueprint.Blueprint, outputType, arch stri excludeSpecs = append(excludePackages, excludeSpecs...) } - packages, checksums, err := api.rpmmd.Depsolve(specs, excludeSpecs, repos, api.distro.ModulePlatformID(), clean) + packages, checksums, err := api.rpmmd.Depsolve(specs, excludeSpecs, repos, api.distro.ModulePlatformID()) if err != nil { return nil, nil, nil, err } @@ -2012,7 +2012,7 @@ func (api *API) depsolveBlueprint(bp *blueprint.Blueprint, outputType, arch stri if err != nil { return nil, nil, nil, err } - buildPackages, _, err = api.rpmmd.Depsolve(buildSpecs, nil, repos, api.distro.ModulePlatformID(), false) + buildPackages, _, err = api.rpmmd.Depsolve(buildSpecs, nil, repos, api.distro.ModulePlatformID()) if err != nil { return nil, nil, nil, err }