particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

weldr: ensure a fresh dnf cache when making a new compose

Browse source 
dnf-json relies on dnf's ability to cache repository metadata. This is
important, because the API calls it quite often to serve requests for
package lists and depsolves.

However, osbuild's dnf stage always fetches new metadata, because it
doesn't have access to the host's cache. Since metadata is valid for
some time, even after a repository changed, the checksum we put in
the pipeline might be old.

Force a new metadata download when producing the pipeline. This is still
not perfect, but greatly reduces the probability of putting stale
metadata into the pipeline.
This commit is contained in:
Lars Karlitski 2019-12-19 20:33:57 +01:00
parent 0ef89aa864
commit 839a109c78
5 changed files with 20 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmd/osbuild-pipeline/main.go
View file

@ -52,7 +52,7 @@ func main() {
}
rpmmd := rpmmd.NewRPMMD()
_, checksums, err := rpmmd.Depsolve(packages, d.Repositories(archArg))
_, checksums, err := rpmmd.Depsolve(packages, d.Repositories(archArg), true)
if err != nil {
panic(err.Error())
}

10
dnf-json
View file

@ -4,6 +4,7 @@ import datetime
import dnf
import hashlib
import json
import shutil
import sys
DNF_ERROR_EXIT_CODE = 10
@ -35,9 +36,12 @@ def dnfrepo(desc, parent_conf=None):
return repo
def create_base(repos):
def create_base(repos, clean=False):
base = dnf.Base()
if clean:
shutil.rmtree(base.conf.cachedir, ignore_errors=True)
for repo in repos:
base.repos.add(dnfrepo(repo, base.conf))
@ -79,7 +83,7 @@ command = call["command"]
arguments = call.get("arguments", {})
if command == "dump":
base = create_base(arguments.get("repos", {}))
base = create_base(arguments.get("repos", {}), arguments.get("clean", False))
packages = []
for package in base.sack.query().available():
packages.append({
@ -100,7 +104,7 @@ if command == "dump":
}, sys.stdout)
elif command == "depsolve":
base = create_base(arguments.get("repos", {}))
base = create_base(arguments.get("repos", {}), arguments.get("clean", False))
errors = []
try:

2
internal/mocks/rpmmd/rpmmd_mock.go
View file

@ -34,6 +34,6 @@ func (r *rpmmdMock) FetchPackageList(repos []rpmmd.RepoConfig) (rpmmd.PackageLis
return r.Fixture.fetchPackageList.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.fetchPackageList.err
}
func (r *rpmmdMock) Depsolve(specs []string, repos []rpmmd.RepoConfig) ([]rpmmd.PackageSpec, map[string]string, error) {
func (r *rpmmdMock) Depsolve(specs []string, repos []rpmmd.RepoConfig, clean bool) ([]rpmmd.PackageSpec, map[string]string, error) {
return r.Fixture.depsolve.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.depsolve.err
}

9
internal/rpmmd/repository.go
View file

@ -94,7 +94,7 @@ type PackageInfo struct {
type RPMMD interface {
FetchPackageList(repos []RepoConfig) (PackageList, map[string]string, error)
Depsolve(specs []string, repos []RepoConfig) ([]PackageSpec, map[string]string, error)
Depsolve(specs []string, repos []RepoConfig, clean bool) ([]PackageSpec, map[string]string, error)
}
type DNFError struct {
@ -187,11 +187,12 @@ func (*rpmmdImpl) FetchPackageList(repos []RepoConfig) (PackageList, map[string]
return reply.Packages, reply.Checksums, err
}
func (*rpmmdImpl) Depsolve(specs []string, repos []RepoConfig) ([]PackageSpec, map[string]string, error) {
func (*rpmmdImpl) Depsolve(specs []string, repos []RepoConfig, clean bool) ([]PackageSpec, map[string]string, error) {
var arguments = struct {
PackageSpecs []string `json:"package-specs"`
Repos []RepoConfig `json:"repos"`
}{specs, repos}
Clean bool `json:"clean,omitempty"`
}{specs, repos, clean}
var reply struct {
Checksums map[string]string `json:"checksums"`
Dependencies []PackageSpec `json:"dependencies"`
@ -253,6 +254,6 @@ func (packages PackageList) ToPackageInfos() []PackageInfo {
}
func (pkg *PackageInfo) FillDependencies(rpmmd RPMMD, repos []RepoConfig) (err error) {
pkg.Dependencies, _, err = rpmmd.Depsolve([]string{pkg.Name}, repos)
pkg.Dependencies, _, err = rpmmd.Depsolve([]string{pkg.Name}, repos, false)
return
}

12
internal/weldr/api.go
View file

@ -644,7 +644,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt
names := strings.Split(params.ByName("projects"), ",")
packages, _, err := api.rpmmd.Depsolve(names, api.distro.Repositories(api.arch))
packages, _, err := api.rpmmd.Depsolve(names, api.distro.Repositories(api.arch), false)
if err != nil {
errors := responseError{
@ -826,7 +826,7 @@ func (api *API) blueprintsDepsolveHandler(writer http.ResponseWriter, request *h
return
}
dependencies, _, err := api.depsolveBlueprint(blueprint)
dependencies, _, err := api.depsolveBlueprint(blueprint, false)
if err != nil {
errors := responseError{
@ -888,7 +888,7 @@ func (api *API) blueprintsFreezeHandler(writer http.ResponseWriter, request *htt
break
}
dependencies, _, err := api.depsolveBlueprint(blueprint)
dependencies, _, err := api.depsolveBlueprint(blueprint, false)
if err != nil {
rerr := responseError{
ID: "BlueprintsError",
@ -1245,7 +1245,7 @@ func (api *API) composeHandler(writer http.ResponseWriter, request *http.Request
bp := api.store.GetBlueprintCommitted(cr.BlueprintName)
if bp != nil {
_, checksums, err := api.depsolveBlueprint(bp)
_, checksums, err := api.depsolveBlueprint(bp, true)
if err != nil {
errors := responseError{
ID: "DepsolveError",
@ -1652,7 +1652,7 @@ func (api *API) fetchPackageList() (rpmmd.PackageList, error) {
return packages, err
}
func (api *API) depsolveBlueprint(bp *blueprint.Blueprint) ([]rpmmd.PackageSpec, map[string]string, error) {
func (api *API) depsolveBlueprint(bp *blueprint.Blueprint, clean bool) ([]rpmmd.PackageSpec, map[string]string, error) {
specs := make([]string, len(bp.Packages))
for i, pkg := range bp.Packages {
specs[i] = pkg.Name
@ -1673,7 +1673,7 @@ func (api *API) depsolveBlueprint(bp *blueprint.Blueprint) ([]rpmmd.PackageSpec,
repos = append(repos, source.RepoConfig())
}
return api.rpmmd.Depsolve(specs, repos)
return api.rpmmd.Depsolve(specs, repos, clean)
}
func (api *API) uploadsScheduleHandler(writer http.ResponseWriter, request *http.Request, params httprouter.Params) {