particle-os/debian-forge-composer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

rpmmd: no longer flush the caches on every call

Browse source 
When we used the dnf-based pipelines, we were relying on the fact
that the metadata was unlikely to have changed between we generated
the pipeline and called osbuild. We achieved this by always updating
to the most recent metadata on every call to rpmmd.Depsolve that
would end up in a pipelin.

Refreshing the metadata is time-consuming, and something we want
to avoid if at all possible. Now that our pipelines no longer
rely on this property, we can drop the flushing.

Signed-off-by: Tom Gundersen <teg@jklm.no>
This commit is contained in:
Tom Gundersen 2020-03-14 20:47:15 +01:00
parent 8b940d09e2
commit 819430e659
5 changed files with 12 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

4
cmd/osbuild-pipeline/main.go
View file

@ -112,7 +112,7 @@ func main() {
} }
rpmmd := rpmmd.NewRPMMD(path.Join(home, ".cache/osbuild-composer/rpmmd")) rpmmd := rpmmd.NewRPMMD(path.Join(home, ".cache/osbuild-composer/rpmmd"))
packageSpecs, checksums, err := rpmmd.Depsolve(packages, exclude_pkgs, d.Repositories(archArg), d.ModulePlatformID(), false) packageSpecs, checksums, err := rpmmd.Depsolve(packages, exclude_pkgs, d.Repositories(archArg), d.ModulePlatformID())
if err != nil { if err != nil {
panic("Could not depsolve: " + err.Error()) panic("Could not depsolve: " + err.Error())
} }
@ -121,7 +121,7 @@ func main() {
if err != nil { if err != nil {
panic("Could not get build packages: " + err.Error()) panic("Could not get build packages: " + err.Error())
} }
buildPackageSpecs, _, err := rpmmd.Depsolve(buildPkgs, nil, d.Repositories(archArg), d.ModulePlatformID(), false) buildPackageSpecs, _, err := rpmmd.Depsolve(buildPkgs, nil, d.Repositories(archArg), d.ModulePlatformID())
if err != nil { if err != nil {
panic("Could not depsolve build packages: " + err.Error()) panic("Could not depsolve build packages: " + err.Error())
} }

8
dnf-json
View file

@ -38,16 +38,13 @@ def dnfrepo(desc, parent_conf=None):
return repo return repo
def create_base(repos, module_platform_id, persistdir, cachedir, clean=False): def create_base(repos, module_platform_id, persistdir, cachedir):
base = dnf.Base() base = dnf.Base()
base.conf.module_platform_id = module_platform_id base.conf.module_platform_id = module_platform_id
base.conf.config_file_path = "/dev/null" base.conf.config_file_path = "/dev/null"
base.conf.persistdir = persistdir base.conf.persistdir = persistdir
base.conf.cachedir = cachedir base.conf.cachedir = cachedir
if clean:
shutil.rmtree(base.conf.cachedir, ignore_errors=True)
for repo in repos: for repo in repos:
base.repos.add(dnfrepo(repo, base.conf)) base.repos.add(dnfrepo(repo, base.conf))
@ -88,12 +85,11 @@ call = json.load(sys.stdin)
command = call["command"] command = call["command"]
arguments = call["arguments"] arguments = call["arguments"]
repos = arguments.get("repos", {}) repos = arguments.get("repos", {})
clean = arguments.get("clean", False)
cachedir = arguments["cachedir"] cachedir = arguments["cachedir"]
module_platform_id = arguments["module_platform_id"] module_platform_id = arguments["module_platform_id"]
with tempfile.TemporaryDirectory() as persistdir: with tempfile.TemporaryDirectory() as persistdir:
base = create_base(repos, module_platform_id, persistdir, cachedir, clean) base = create_base(repos, module_platform_id, persistdir, cachedir)
if command == "dump": if command == "dump":
packages = [] packages = []

2
internal/mocks/rpmmd/rpmmd_mock.go
View file

@ -34,6 +34,6 @@ func (r *rpmmdMock) FetchMetadata(repos []rpmmd.RepoConfig, modulePlatformID str
return r.Fixture.fetchPackageList.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.fetchPackageList.err return r.Fixture.fetchPackageList.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.fetchPackageList.err
} }
func (r *rpmmdMock) Depsolve(specs, excludeSpecs []string, repos []rpmmd.RepoConfig, modulePlatformID string, clean bool) ([]rpmmd.PackageSpec, map[string]string, error) { func (r *rpmmdMock) Depsolve(specs, excludeSpecs []string, repos []rpmmd.RepoConfig, modulePlatformID string) ([]rpmmd.PackageSpec, map[string]string, error) {
return r.Fixture.depsolve.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.depsolve.err return r.Fixture.depsolve.ret, r.Fixture.fetchPackageList.checksums, r.Fixture.depsolve.err
} }

9
internal/rpmmd/repository.go
View file

@ -102,7 +102,7 @@ type RPMMD interface {
// Depsolve takes a list of required content (specs), explicitly unwanted content (excludeSpecs), list // Depsolve takes a list of required content (specs), explicitly unwanted content (excludeSpecs), list
// or repositories, and platform ID for modularity. It returns a list of all packages (with solved // or repositories, and platform ID for modularity. It returns a list of all packages (with solved
// dependencies) that will be installed into the system. // dependencies) that will be installed into the system.
Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID string, clean bool) ([]PackageSpec, map[string]string, error) Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID string) ([]PackageSpec, map[string]string, error)
} }
type DNFError struct { type DNFError struct {
@ -237,15 +237,14 @@ func (r *rpmmdImpl) FetchMetadata(repos []RepoConfig, modulePlatformID string) (
return reply.Packages, reply.Checksums, err return reply.Packages, reply.Checksums, err
} }
func (r *rpmmdImpl) Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID string, clean bool) ([]PackageSpec, map[string]string, error) { func (r *rpmmdImpl) Depsolve(specs, excludeSpecs []string, repos []RepoConfig, modulePlatformID string) ([]PackageSpec, map[string]string, error) {
var arguments = struct { var arguments = struct {
PackageSpecs []string `json:"package-specs"` PackageSpecs []string `json:"package-specs"`
ExcludSpecs []string `json:"exclude-specs"` ExcludSpecs []string `json:"exclude-specs"`
Repos []RepoConfig `json:"repos"` Repos []RepoConfig `json:"repos"`
CacheDir string `json:"cachedir"` CacheDir string `json:"cachedir"`
ModulePlatformID string `json:"module_platform_id"` ModulePlatformID string `json:"module_platform_id"`
Clean bool `json:"clean,omitempty"` }{specs, excludeSpecs, repos, r.CacheDir, modulePlatformID}
}{specs, excludeSpecs, repos, r.CacheDir, modulePlatformID, clean}
var reply struct { var reply struct {
Checksums map[string]string `json:"checksums"` Checksums map[string]string `json:"checksums"`
Dependencies []PackageSpec `json:"dependencies"` Dependencies []PackageSpec `json:"dependencies"`
@ -307,6 +306,6 @@ func (packages PackageList) ToPackageInfos() []PackageInfo {
} }
func (pkg *PackageInfo) FillDependencies(rpmmd RPMMD, repos []RepoConfig, modulePlatformID string) (err error) { func (pkg *PackageInfo) FillDependencies(rpmmd RPMMD, repos []RepoConfig, modulePlatformID string) (err error) {
pkg.Dependencies, _, err = rpmmd.Depsolve([]string{pkg.Name}, nil, repos, modulePlatformID, false) pkg.Dependencies, _, err = rpmmd.Depsolve([]string{pkg.Name}, nil, repos, modulePlatformID)
return return
} }

6
internal/weldr/api.go
View file

@ -653,7 +653,7 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt
names := strings.Split(params.ByName("projects"), ",") names := strings.Split(params.ByName("projects"), ",")
packages, _, err := api.rpmmd.Depsolve(names, nil, api.distro.Repositories(api.arch), api.distro.ModulePlatformID(), false) packages, _, err := api.rpmmd.Depsolve(names, nil, api.distro.Repositories(api.arch), api.distro.ModulePlatformID())
if err != nil { if err != nil {
errors := responseError{ errors := responseError{
@ -2001,7 +2001,7 @@ func (api *API) depsolveBlueprint(bp *blueprint.Blueprint, outputType, arch stri
excludeSpecs = append(excludePackages, excludeSpecs...) excludeSpecs = append(excludePackages, excludeSpecs...)
} }
packages, checksums, err := api.rpmmd.Depsolve(specs, excludeSpecs, repos, api.distro.ModulePlatformID(), clean) packages, checksums, err := api.rpmmd.Depsolve(specs, excludeSpecs, repos, api.distro.ModulePlatformID())
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, nil, err
} }
@ -2012,7 +2012,7 @@ func (api *API) depsolveBlueprint(bp *blueprint.Blueprint, outputType, arch stri
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, nil, err
} }
buildPackages, _, err = api.rpmmd.Depsolve(buildSpecs, nil, repos, api.distro.ModulePlatformID(), false) buildPackages, _, err = api.rpmmd.Depsolve(buildSpecs, nil, repos, api.distro.ModulePlatformID())
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, nil, err
} }