dnfjson: Cleanup old distro cache dirs

This adds a function, CleanupOldCacheDirs, that checks the dirs under
/var/cache/osbuild-composer/rpmmd/ and removes files and directories
that don't match the current list of supported distros.

This will clean up the cache from old releases as the are retired, and
will also cleanup the old top level cache directory structure after an
upgrade.

NOTE: This function does not return errors, any real problems it
encounters will also be caught by the cache initialization code and
handled there.

cmd/osbuild-composer/composer.go

@@ -71,6 +71,9 @@ func NewComposer(config *ComposerConfigFile, stateDir, cacheDir string) (*Compos
 	c.distros = distroregistry.NewDefault()
 	logrus.Infof("Loaded %d distros", len(c.distros.List()))
 
+	// Clean up the cache, removes unknown distros and files
+	dnfjson.CleanupOldCacheDirs(path.Join(c.cacheDir, "rpmmd"), c.distros.List())
+
 	c.solver = dnfjson.NewBaseSolver(path.Join(c.cacheDir, "rpmmd"))
 	c.solver.SetDNFJSONPath(c.config.DNFJson)
 

internal/dnfjson/cache.go

@@ -14,6 +14,44 @@ import (
 	"github.com/gobwas/glob"
 )
 
+// CleanupOldCacheDirs will remove cache directories for unsupported distros
+// eg. Once support for a fedora release stops and it is removed, this will
+// delete its directory under root.
+//
+// A happy side effect of this is that it will delete old cache directories
+// and files from before the switch to per-distro cache directories.
+//
+// NOTE: This does not return any errors. This is because the most common one
+// will be a nonexistant directory which will be created later, during initial
+// cache creation. Any other errors like permission issues will be caught by
+// later use of the cache. eg. touchRepo
+func CleanupOldCacheDirs(root string, distros []string) {
+	dirs, _ := os.ReadDir(root)
+
+	for _, e := range dirs {
+		if strSliceContains(distros, e.Name()) {
+			// known distro
+			continue
+		}
+		if e.IsDir() {
+			// Remove the directory and everything under it
+			_ = os.RemoveAll(filepath.Join(root, e.Name()))
+		} else {
+			_ = os.Remove(filepath.Join(root, e.Name()))
+		}
+	}
+}
+
+// strSliceContains returns true if the elem string is in the slc array
+func strSliceContains(slc []string, elem string) bool {
+	for _, s := range slc {
+		if elem == s {
+			return true
+		}
+	}
+	return false
+}
+
 // global cache locker
 var cacheLocks sync.Map
 
@@ -69,8 +107,12 @@ func newRPMCache(path string, maxSize uint64) *rpmCache {
 }
 
 // updateInfo updates the repoPaths and repoRecency fields of the rpmCache.
+//
+// NOTE: This does not return any errors. This is because the most common one
+// will be a nonexistant directory which will be created later, during initial
+// cache creation. Any other errors like permission issues will be caught by
+// later use of the cache. eg. touchRepo
 func (r *rpmCache) updateInfo() {
-	// Top level of the cache is now used for separate distributions
 	dirs, _ := os.ReadDir(r.root)
 	for _, d := range dirs {
 		r.updateCacheDirInfo(filepath.Join(r.root, d.Name()))
@@ -78,6 +120,7 @@ func (r *rpmCache) updateInfo() {
 }
 
 func (r *rpmCache) updateCacheDirInfo(path string) {
+	// See updateInfo NOTE on error handling
 	cacheEntries, _ := os.ReadDir(path)
 
 	// each repository has multiple cache entries (3 on average), so using the

internal/dnfjson/cache_test.go

@@ -4,6 +4,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -171,15 +172,6 @@ func TestCacheRead(t *testing.T) {
 	}
 }
 
-func strSliceContains(slc []string, elem string) bool {
-	for _, s := range slc {
-		if elem == s {
-			return true
-		}
-	}
-	return false
-}
-
 func sizeSum(cfg testCache, repoIDFilter ...string) uint64 {
 	var sum uint64
 	for path, info := range cfg {
@@ -329,3 +321,64 @@ func TestDNFCacheCleanup(t *testing.T) {
 	_, ok := cache.Get("notreallyahash")
 	assert.False(t, ok)
 }
+
+func TestCleanupOldCacheDirs(t *testing.T) {
+	// Run the cleanup without the cache present and with dummy distro names
+	CleanupOldCacheDirs("/var/tmp/test-no-cache-rpmmd/", []string{"fedora-37", "fedora-38"})
+
+	testCacheRoot := t.TempDir()
+	// Make all the test caches under root, using their keys as a distro name.
+	var distros []string
+	for name, cfg := range testCfgs {
+		// Cache is now per-distro, use the name of the config as a distro name
+		createTestCache(filepath.Join(testCacheRoot, name), cfg)
+		distros = append(distros, name)
+	}
+	sort.Strings(distros)
+
+	// Add the content of the 'fake-real' cache to the top directory
+	// this will be used to simulate an old cache without distro subdirs
+	createTestCache(testCacheRoot, testCfgs["fake-real"])
+
+	CleanupOldCacheDirs(testCacheRoot, distros)
+
+	// The fake-real files under the root directory should all be gone.
+	for path := range testCfgs["fake-real"] {
+		_, err := os.Stat(filepath.Join(testCacheRoot, path))
+		assert.NotNil(t, err)
+	}
+
+	// The distro cache files should all still be present
+	for name, cfg := range testCfgs {
+		for path := range cfg {
+			_, err := os.Stat(filepath.Join(testCacheRoot, name, path))
+			assert.Nil(t, err)
+		}
+	}
+
+	// Remove the fake-real distro from the list
+	// This simulates retiring an older distribution and cleaning up its cache
+	distros = []string{}
+	for name := range testCfgs {
+		if name == "fake-real" {
+			continue
+		}
+		distros = append(distros, name)
+	}
+	// Cleanup should now remove the fake-real subdirectory and files
+	CleanupOldCacheDirs(testCacheRoot, distros)
+
+	// The remaining distro's cache files should all still be present
+	for _, name := range distros {
+		for path := range testCfgs[name] {
+			_, err := os.Stat(filepath.Join(testCacheRoot, name, path))
+			assert.Nil(t, err)
+		}
+	}
+
+	// But the fake-real ones should be gone
+	for path := range testCfgs["fake-real"] {
+		_, err := os.Stat(filepath.Join(testCacheRoot, "fake-real", path))
+		assert.NotNil(t, err)
+	}
+}