diff --git a/cmd/osbuild-pipeline/main.go b/cmd/osbuild-pipeline/main.go index 7eb496570..a4b076da8 100644 --- a/cmd/osbuild-pipeline/main.go +++ b/cmd/osbuild-pipeline/main.go @@ -143,6 +143,11 @@ func main() { solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch.Name(), path.Join(home, ".cache/osbuild-composer/rpmmd")) solver.SetDNFJSONPath(findDnfJsonBin()) + // Set cache size to 3 GiB + // osbuild-pipeline is often used to generate a lot of manifests in a row + // let the cache grow to fit much more repository metadata than we usually allow + solver.SetMaxCacheSize(3 * 1024 * 1024 * 1024) + packageSets := imageType.PackageSets(composeRequest.Blueprint, repos) depsolvedSets := make(map[string][]rpmmd.PackageSpec) @@ -189,4 +194,8 @@ func main() { } } os.Stdout.Write(bytes) + if err := solver.CleanCache(); err != nil { + // print to stderr but don't exit with error + fmt.Fprintf(os.Stderr, "Error during rpm repo cache cleanup: %s", err.Error()) + } } diff --git a/cmd/osbuild-worker/jobimpl-depsolve.go b/cmd/osbuild-worker/jobimpl-depsolve.go index e770d43ca..e1fcd2ee5 100644 --- a/cmd/osbuild-worker/jobimpl-depsolve.go +++ b/cmd/osbuild-worker/jobimpl-depsolve.go @@ -30,6 +30,7 @@ func (impl *DepsolveJobImpl) depsolve(packageSets map[string][]rpmmd.PackageSet, } depsolvedSets[name] = res } + return depsolvedSets, nil } @@ -66,6 +67,10 @@ func (impl *DepsolveJobImpl) Run(job worker.Job) error { logWithId.Errorf("rpmmd error in depsolve job: %v", err) } } + if err := impl.Solver.CleanCache(); err != nil { + // log and ignore + logWithId.Errorf("Error during rpm repo cache cleanup: %s", err.Error()) + } err = job.Update(&result) if err != nil { diff --git a/internal/dnfjson/cache.go b/internal/dnfjson/cache.go new file mode 100644 index 000000000..926fdca1d --- /dev/null +++ b/internal/dnfjson/cache.go @@ -0,0 +1,31 @@ +package dnfjson + +import ( + "io/fs" + "os" + "path/filepath" +) + +func dirSize(path string) (uint64, error) { + var size uint64 + sizer := func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + size += uint64(info.Size()) + return nil + } + err := filepath.Walk(path, sizer) + return size, err +} + +func (bs *BaseSolver) CleanCache() error { + curSize, err := dirSize(bs.cacheDir) + if err != nil { + return err + } + if curSize > bs.maxCacheSize { + return os.RemoveAll(bs.cacheDir) + } + return nil +} diff --git a/internal/dnfjson/dnfjson.go b/internal/dnfjson/dnfjson.go index 8cfd1c8ad..223b55c7a 100644 --- a/internal/dnfjson/dnfjson.go +++ b/internal/dnfjson/dnfjson.go @@ -34,6 +34,8 @@ type BaseSolver struct { // Path to the dnf-json binary and optional args (default: "/usr/libexec/osbuild-composer/dnf-json") dnfJsonCmd []string + + maxCacheSize uint64 } // Create a new unconfigured BaseSolver (without platform information). It can @@ -45,9 +47,14 @@ func NewBaseSolver(cacheDir string) *BaseSolver { cacheDir: cacheDir, subscriptions: subscriptions, dnfJsonCmd: []string{"/usr/libexec/osbuild-composer/dnf-json"}, + maxCacheSize: 524288000, // 500 MiB } } +func (s *BaseSolver) SetMaxCacheSize(size uint64) { + s.maxCacheSize = size +} + // SetDNFJSONPath sets the path to the dnf-json binary and optionally any command line arguments. func (s *BaseSolver) SetDNFJSONPath(cmd string, args ...string) { s.dnfJsonCmd = append([]string{cmd}, args...) diff --git a/internal/kojiapi/server.go b/internal/kojiapi/server.go index 155b51389..8d0499cfc 100644 --- a/internal/kojiapi/server.go +++ b/internal/kojiapi/server.go @@ -224,6 +224,11 @@ func (h *apiHandlers) PostCompose(ctx echo.Context) error { return echo.NewHTTPError(http.StatusBadRequest, fmt.Sprintf("Could not initialize build with koji: %v", initResult.JobError.Reason)) } + if err := h.server.solver.CleanCache(); err != nil { + // log and ignore + log.Printf("Error during rpm repo cache cleanup: %s", err.Error()) + } + return ctx.JSON(http.StatusCreated, &api.ComposeResponse{ Id: id.String(), KojiBuildId: int(initResult.BuildID), diff --git a/internal/weldr/api.go b/internal/weldr/api.go index 332cc37d8..8be0cd220 100644 --- a/internal/weldr/api.go +++ b/internal/weldr/api.go @@ -1272,6 +1272,10 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req } packageInfos[i].Dependencies = solved } + if err := solver.CleanCache(); err != nil { + // log and ignore + log.Printf("Error during rpm repo cache cleanup: %s", err.Error()) + } } if modulesRequested { @@ -1348,6 +1352,10 @@ func (api *API) projectsDepsolveHandler(writer http.ResponseWriter, request *htt statusResponseError(writer, http.StatusBadRequest, errors) return } + if err := solver.CleanCache(); err != nil { + // log and ignore + log.Printf("Error during rpm repo cache cleanup: %s", err.Error()) + } err = json.NewEncoder(writer).Encode(reply{Projects: deps}) common.PanicOnError(err) } @@ -2148,6 +2156,10 @@ func (api *API) depsolveBlueprintForImageType(bp blueprint.Blueprint, imageType } depsolvedSets[name] = res } + if err := solver.CleanCache(); err != nil { + // log and ignore + log.Printf("Error during rpm repo cache cleanup: %s", err.Error()) + } return depsolvedSets, nil } @@ -3120,6 +3132,10 @@ func (api *API) fetchPackageList(distroName string) (rpmmd.PackageList, error) { if err != nil { return nil, err } + if err := solver.CleanCache(); err != nil { + // log and ignore + log.Printf("Error during rpm repo cache cleanup: %s", err.Error()) + } return packages, nil } @@ -3190,6 +3206,10 @@ func (api *API) depsolveBlueprint(bp blueprint.Blueprint) ([]rpmmd.PackageSpec, return nil, err } + if err := solver.CleanCache(); err != nil { + // log and ignore + log.Printf("Error during rpm repo cache cleanup: %s", err.Error()) + } return solved, nil }