From 839a109c78e25ce7e9f217cbec159b2b7af80b35 Mon Sep 17 00:00:00 2001 From: Lars Karlitski Date: Thu, 19 Dec 2019 20:33:57 +0100 Subject: [PATCH] weldr: ensure a fresh dnf cache when making a new compose dnf-json relies on dnf's ability to cache repository metadata. This is important, because the API calls it quite often to serve requests for package lists and depsolves. However, osbuild's dnf stage always fetches new metadata, because it doesn't have access to the host's cache. Since metadata is valid for some time, even after a repository changed, the checksum we put in the pipeline might be old. Force a new metadata download when producing the pipeline. This is still not perfect, but greatly reduces the probability of putting stale metadata into the pipeline. --- cmd/osbuild-pipeline/main.go | 2 +- dnf-json | 10 +++++++--- internal/mocks/rpmmd/rpmmd_mock.go | 2 +- internal/rpmmd/repository.go | 9 +++++---- internal/weldr/api.go | 12 ++++++------ 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go index 9fb4e3013..5b03dcce2 100644 --- a/cmd/osbuild-pipeline/main.go +++ b/cmd/osbuild-pipeline/main.go @@ -52,7 +52,7 @@ func main() { } rpmmd := rpmmd.NewRPMMD() - _, checksums, err := rpmmd.Depsolve(packages, d.Repositories(archArg)) + _, checksums, err := rpmmd.Depsolve(packages, d.Repositories(archArg), true) if err != nil { panic(err.Error()) } diff --git a/dnf-json b/dnf-json index 9ff287a14..80a4981ad 100644 --- a/dnf-json +++ b/dnf-json @@ -4,6 +4,7 @@ import datetime import dnf import hashlib import json +import shutil import sys DNF_ERROR_EXIT_CODE = 10 @@ -35,9 +36,12 @@ def dnfrepo(desc, parent_conf=None): return repo -def create_base(repos): +def create_base(repos, clean=False): base = dnf.Base() + if clean: + shutil.rmtree(base.conf.cachedir, ignore_errors=True) + for repo in repos: base.repos.add(dnfrepo(repo, base.conf)) @@ -79,7 +83,7 @@ command = call["command"] arguments = call.get("arguments", {}) if command == "dump": - base = create_base(arguments.get("repos", {})) + base = create_base(arguments.get("repos", {}), arguments.get("clean", False)) packages = [] for package in base.sack.query().available(): packages.append({ @@ -100,7 +104,7 @@ if command == "dump": }, sys.stdout) elif command == "depsolve": - base = create_base(arguments.get("repos", {})) + base = create_base(arguments.get("repos", {}), arguments.get("clean", False)) errors = [] try: diff --git a/internal/mocks/rpmmd/rpmmd_mock.go b/internal/mocks/rpmmd/rpmmd_mock.go index 229703837..d5d3162fb 100644 --- a/internal/mocks/rpmmd/rpmmd_mock.go +++ b/internal/mocks/rpmmd/rpmmd_mock.go @@ -34,6 +34,6 @@ func (r *rpmmdMock) FetchPackageList(repos []rpmmd.RepoConfig) (rpmmd.PackageLis return r.Fixture.fetchPackageList.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.fetchPackageList.err } -func (r *rpmmdMock) Depsolve(specs []string, repos []rpmmd.RepoConfig) ([]rpmmd.PackageSpec, map[string]string, error) { +func (r *rpmmdMock) Depsolve(specs []string, repos []rpmmd.RepoConfig, clean bool) ([]rpmmd.PackageSpec, map[string]string, error) { return r.Fixture.depsolve.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.depsolve.err } diff --git a/internal/rpmmd/repository.go b/internal/rpmmd/repository.go index a3e46a42a..f976f238f 100644 --- a/internal/rpmmd/repository.go +++ b/internal/rpmmd/repository.go @@ -94,7 +94,7 @@ type PackageInfo struct { type RPMMD interface { FetchPackageList(repos []RepoConfig) (PackageList, map[string]string, error) - Depsolve(specs []string, repos []RepoConfig) ([]PackageSpec, map[string]string, error) + Depsolve(specs []string, repos []RepoConfig, clean bool) ([]PackageSpec, map[string]string, error) } type DNFError struct { @@ -187,11 +187,12 @@ func (*rpmmdImpl) FetchPackageList(repos []RepoConfig) (PackageList, map[string] return reply.Packages, reply.Checksums, err } -func (*rpmmdImpl) Depsolve(specs []string, repos []RepoConfig) ([]PackageSpec, map[string]string, error) { +func (*rpmmdImpl) Depsolve(specs []string, repos []RepoConfig, clean bool) ([]PackageSpec, map[string]string, error) { var arguments = struct { PackageSpecs []string `json:"package-specs"` Repos []RepoConfig `json:"repos"` - }{specs, repos} + Clean bool `json:"clean,omitempty"` + }{specs, repos, clean} var reply struct { Checksums map[string]string `json:"checksums"` Dependencies []PackageSpec `json:"dependencies"` @@ -253,6 +254,6 @@ func (packages PackageList) ToPackageInfos() []PackageInfo { } func (pkg *PackageInfo) FillDependencies(rpmmd RPMMD, repos []RepoConfig) (err error) { - pkg.Dependencies, _, err = rpmmd.Depsolve([]string{pkg.Name}, repos) + pkg.Dependencies, _, err = rpmmd.Depsolve([]string{pkg.Name}, repos, false) return } diff --git a/internal/weldr/api.go b/internal/weldr/api.go index ca6e75795..b0694f781 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -644,7 +644,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt names := strings.Split(params.ByName("projects"), ",") - packages, _, err := api.rpmmd.Depsolve(names, api.distro.Repositories(api.arch)) + packages, _, err := api.rpmmd.Depsolve(names, api.distro.Repositories(api.arch), false) if err != nil { errors := responseError{ @@ -826,7 +826,7 @@ func (api *API) blueprintsDepsolveHandler(writer http.ResponseWriter, request *h return } - dependencies, _, err := api.depsolveBlueprint(blueprint) + dependencies, _, err := api.depsolveBlueprint(blueprint, false) if err != nil { errors := responseError{ @@ -888,7 +888,7 @@ func (api *API) blueprintsFreezeHandler(writer http.ResponseWriter, request *htt break } - dependencies, _, err := api.depsolveBlueprint(blueprint) + dependencies, _, err := api.depsolveBlueprint(blueprint, false) if err != nil { rerr := responseError{ ID: "BlueprintsError", @@ -1245,7 +1245,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request bp := api.store.GetBlueprintCommitted(cr.BlueprintName) if bp != nil { - _, checksums, err := api.depsolveBlueprint(bp) + _, checksums, err := api.depsolveBlueprint(bp, true) if err != nil { errors := responseError{ ID: "DepsolveError", @@ -1652,7 +1652,7 @@ func (api *API) fetchPackageList() (rpmmd.PackageList, error) { return packages, err } -func (api *API) depsolveBlueprint(bp *blueprint.Blueprint) ([]rpmmd.PackageSpec, map[string]string, error) { +func (api *API) depsolveBlueprint(bp *blueprint.Blueprint, clean bool) ([]rpmmd.PackageSpec, map[string]string, error) { specs := make([]string, len(bp.Packages)) for i, pkg := range bp.Packages { specs[i] = pkg.Name @@ -1673,7 +1673,7 @@ func (api *API) depsolveBlueprint(bp *blueprint.Blueprint) ([]rpmmd.PackageSpec, repos = append(repos, source.RepoConfig()) } - return api.rpmmd.Depsolve(specs, repos) + return api.rpmmd.Depsolve(specs, repos, clean) } func (api *API) uploadsScheduleHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {