f731ab53d0
This adds a function, CleanupOldCacheDirs, that checks the dirs under /var/cache/osbuild-composer/rpmmd/ and removes files and directories that don't match the current list of supported distros. This will clean up the cache from old releases as the are retired, and will also cleanup the old top level cache directory structure after an upgrade. NOTE: This function does not return errors, any real problems it encounters will also be caught by the cache initialization code and handled there.
384 lines
16 KiB
Go
384 lines
16 KiB
Go
package dnfjson
|
|
|
|
import (
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/osbuild/osbuild-composer/internal/rpmmd"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func truncate(path string, size int64) {
|
|
fp, err := os.Create(path)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
defer fp.Close()
|
|
if err := fp.Truncate(size); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// create a test cache based on the config, where the config keys are file
|
|
// paths and the values are file sizes
|
|
func createTestCache(root string, config testCache) uint64 {
|
|
var totalSize uint64
|
|
for path, fi := range config {
|
|
fullPath := filepath.Join(root, path)
|
|
parPath := filepath.Dir(fullPath)
|
|
if err := os.MkdirAll(parPath, 0770); err != nil {
|
|
panic(err)
|
|
}
|
|
truncate(fullPath, int64(fi.size))
|
|
mtime := time.Unix(fi.mtime, 0)
|
|
if err := os.Chtimes(fullPath, mtime, mtime); err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
// if the path has multiple parts, touch the top level directory of the
|
|
// element
|
|
pathParts := strings.Split(path, "/")
|
|
if len(pathParts) > 1 {
|
|
top := pathParts[0]
|
|
if err := os.Chtimes(filepath.Join(root, top), mtime, mtime); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
if len(path) >= 64 {
|
|
// paths with shorter names will be ignored by the cache manager
|
|
totalSize += fi.size
|
|
}
|
|
}
|
|
|
|
// add directory sizes to total
|
|
sizer := func(path string, info fs.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if path == root {
|
|
// don't count root
|
|
return nil
|
|
}
|
|
if info.IsDir() {
|
|
totalSize += uint64(info.Size())
|
|
}
|
|
return nil
|
|
}
|
|
if err := filepath.Walk(root, sizer); err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
return totalSize
|
|
}
|
|
|
|
type fileInfo struct {
|
|
size uint64
|
|
mtime int64
|
|
}
|
|
|
|
type testCache map[string]fileInfo
|
|
|
|
var testCfgs = map[string]testCache{
|
|
"rhel84-aarch64": { // real repo metadata file names and sizes
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc.solv": fileInfo{2095095, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-filenames.solvx": fileInfo{14473401, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/gen/groups.xml": fileInfo{1419587, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/3eabd1122210e4def18ae4b96a18aa5bcc186abf2ec14e2e8f1c1bb1ab4d11da-modules.yaml.gz": fileInfo{156314, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/90fd2e7463220a07457e76ae905e1bad754c29e22202bb3202c971a5ece28396-comps-AppStream.aarch64.xml.gz": fileInfo{199426, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/77a66c76b5f6ba51aaee6c0cf76d701601e8b622d1701d1781dabec434f27413-filelists.xml.gz": fileInfo{14370201, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/1941c723c94218eed43eac3174aa94cefbe921e15547c39251a95895024207ca-primary.xml.gz": fileInfo{11439375, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/repomd.xml": fileInfo{13285, 100},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62.solv": fileInfo{1147863, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-filenames.solvx": fileInfo{11133964, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-98177081b9162766/repodata/gen/groups.xml": fileInfo{1298102, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-98177081b9162766/repodata/d74783221709ab27d543c1cfc4c02562fde6edfaaaac33ac73a68ecf53188695-comps-BaseOS.aarch64.xml.gz": fileInfo{174076, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-98177081b9162766/repodata/5ded48b4c9e238288130c6670d99f5febdb7273e4a31ac213836a15a2076514d-filelists.xml.gz": fileInfo{11081612, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-98177081b9162766/repodata/8120caf8ebbb8c8b37f6f0dd027d866020ebe7acf9c9ce49ae9903b761986f0c-primary.xml.gz": fileInfo{1836471, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-98177081b9162766/repodata/repomd.xml": fileInfo{12817, 300},
|
|
},
|
|
"fake-real": { // fake but resembling real data
|
|
"1111111111111111111111111111111111111111111111111111111111111111.solv": fileInfo{100, 0},
|
|
"1111111111111111111111111111111111111111111111111111111111111111-filenames.solv": fileInfo{200, 0},
|
|
"1111111111111111111111111111111111111111111111111111111111111111.whatever": fileInfo{110, 0},
|
|
"1111111111111111111111111111111111111111111111111111111111111111/repodata/a": fileInfo{1000, 0},
|
|
"1111111111111111111111111111111111111111111111111111111111111111/repodata/b": fileInfo{3829, 0},
|
|
"1111111111111111111111111111111111111111111111111111111111111111/repodata/c": fileInfo{831989, 0},
|
|
"2222222222222222222222222222222222222222222222222222222222222222.solv": fileInfo{120, 2},
|
|
"2222222222222222222222222222222222222222222222222222222222222222-filenames.solv": fileInfo{232, 2},
|
|
"2222222222222222222222222222222222222222222222222222222222222222.whatever": fileInfo{110, 2},
|
|
"2222222222222222222222222222222222222222222222222222222222222222/repodata/a": fileInfo{1000, 2},
|
|
"2222222222222222222222222222222222222222222222222222222222222222/repodata/b": fileInfo{3829, 2},
|
|
"2222222222222222222222222222222222222222222222222222222222222222/repodata/c": fileInfo{831989, 2},
|
|
"3333333333333333333333333333333333333333333333333333333333333333.solv": fileInfo{105, 4},
|
|
"3333333333333333333333333333333333333333333333333333333333333333-filenames.solv": fileInfo{200, 4},
|
|
"3333333333333333333333333333333333333333333333333333333333333333.whatever": fileInfo{110, 4},
|
|
"3333333333333333333333333333333333333333333333333333333333333333/repodata/a": fileInfo{2390, 4},
|
|
"3333333333333333333333333333333333333333333333333333333333333333/repodata/b": fileInfo{1234890, 4},
|
|
"3333333333333333333333333333333333333333333333333333333333333333/repodata/c": fileInfo{483, 4},
|
|
},
|
|
"completely-fake": { // just a mess of files (including files without a repo ID)
|
|
"somefile": fileInfo{192, 10291920},
|
|
"yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy-repofiley": fileInfo{29384, 11},
|
|
"yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy-repofiley2": fileInfo{293, 31},
|
|
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-repofile": fileInfo{29384, 30},
|
|
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-repofileb": fileInfo{293, 45},
|
|
},
|
|
}
|
|
|
|
type testCase struct {
|
|
cache testCache
|
|
maxSize uint64
|
|
minSizeAfterShrink uint64
|
|
repoIDsAfterShrink []string
|
|
}
|
|
|
|
func getRepoIDs(ct testCache) []string {
|
|
idMap := make(map[string]bool)
|
|
ids := make([]string, 0)
|
|
for path := range ct {
|
|
if len(path) >= 64 {
|
|
id := path[:64]
|
|
if !idMap[id] {
|
|
idMap[id] = true
|
|
ids = append(ids, id)
|
|
}
|
|
}
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func TestCacheRead(t *testing.T) {
|
|
assert := assert.New(t)
|
|
for name, cfg := range testCfgs {
|
|
t.Run(name, func(t *testing.T) {
|
|
testCacheRoot := t.TempDir()
|
|
// Cache is now per-distro, use the name of the config as a distro name
|
|
s := createTestCache(filepath.Join(testCacheRoot, name), cfg)
|
|
|
|
// Cache covers all distros, pass in top directory
|
|
cache := newRPMCache(testCacheRoot, 1048576) // 1 MiB, but doesn't matter for this test
|
|
|
|
nrepos := len(getRepoIDs(cfg))
|
|
assert.Equal(s, cache.size)
|
|
assert.Equal(nrepos, len(cache.repoElements))
|
|
assert.Equal(nrepos, len(cache.repoRecency))
|
|
})
|
|
}
|
|
}
|
|
|
|
func sizeSum(cfg testCache, repoIDFilter ...string) uint64 {
|
|
var sum uint64
|
|
for path, info := range cfg {
|
|
if len(path) < 64 {
|
|
continue
|
|
}
|
|
rid := path[:64]
|
|
if len(repoIDFilter) == 0 || (len(repoIDFilter) > 0 && strSliceContains(repoIDFilter, rid)) {
|
|
sum += info.size
|
|
}
|
|
}
|
|
return sum
|
|
}
|
|
|
|
func TestCacheCleanup(t *testing.T) {
|
|
rhelRecentRepoSize := sizeSum(testCfgs["rhel84-aarch64"], "df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62")
|
|
rhelTotalRepoSize := sizeSum(testCfgs["rhel84-aarch64"])
|
|
|
|
fakeRealSize2 := sizeSum(testCfgs["fake-real"], "2222222222222222222222222222222222222222222222222222222222222222")
|
|
fakeRealSize3 := sizeSum(testCfgs["fake-real"], "3333333333333333333333333333333333333333333333333333333333333333")
|
|
|
|
testCases := map[string]testCase{
|
|
// max size 1 byte -> clean will delete everything
|
|
"fake-real-full-delete": {
|
|
cache: testCfgs["fake-real"],
|
|
maxSize: 1,
|
|
minSizeAfterShrink: 0,
|
|
},
|
|
"rhel-full-delete": {
|
|
cache: testCfgs["rhel84-aarch64"],
|
|
maxSize: 1,
|
|
minSizeAfterShrink: 0,
|
|
},
|
|
"completely-fake-full-delete": {
|
|
cache: testCfgs["completely-fake"],
|
|
maxSize: 1,
|
|
minSizeAfterShrink: 0,
|
|
},
|
|
"completely-fake-full-delete-2": {
|
|
cache: testCfgs["completely-fake"],
|
|
maxSize: 100,
|
|
minSizeAfterShrink: 0,
|
|
},
|
|
// max size a bit larger than most recent repo -> clean will delete older repos
|
|
"completely-fake-half-delete": {
|
|
cache: testCfgs["completely-fake"],
|
|
maxSize: 29384 + 293 + 1, // one byte larger than the files of one repo
|
|
minSizeAfterShrink: 29384 + 293, // size of files from one repo
|
|
repoIDsAfterShrink: []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, // most recent repo timestamp (45)
|
|
},
|
|
"rhel-half-delete": {
|
|
cache: testCfgs["rhel84-aarch64"],
|
|
maxSize: rhelRecentRepoSize + 102400, // most recent repo file sizes + 100k buffer (for directories)
|
|
minSizeAfterShrink: rhelRecentRepoSize, // after shrink it should be at least as big as the most recent repo
|
|
repoIDsAfterShrink: []string{"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62"}, // most recent repo timestamp (45)
|
|
},
|
|
"fake-real-delete-1": {
|
|
cache: testCfgs["fake-real"],
|
|
maxSize: fakeRealSize3 + fakeRealSize2 + 102400,
|
|
minSizeAfterShrink: fakeRealSize3 + fakeRealSize2,
|
|
repoIDsAfterShrink: []string{"3333333333333333333333333333333333333333333333333333333333333333", "2222222222222222222222222222222222222222222222222222222222222222"},
|
|
},
|
|
"fake-real-delete-2": {
|
|
cache: testCfgs["fake-real"],
|
|
maxSize: fakeRealSize3 + 102400,
|
|
minSizeAfterShrink: fakeRealSize3,
|
|
repoIDsAfterShrink: []string{"3333333333333333333333333333333333333333333333333333333333333333"},
|
|
},
|
|
// max size is huge -> clean wont delete anything
|
|
"rhel-no-delete": {
|
|
cache: testCfgs["rhel84-aarch64"],
|
|
maxSize: 45097156608, // 42 GiB
|
|
minSizeAfterShrink: rhelTotalRepoSize,
|
|
repoIDsAfterShrink: []string{"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62", "9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc"},
|
|
},
|
|
}
|
|
|
|
for name, cfg := range testCases {
|
|
t.Run(name, func(t *testing.T) {
|
|
assert := assert.New(t)
|
|
testCacheRoot := t.TempDir()
|
|
// Cache is now per-distro, use the name of the config as a distro name
|
|
createTestCache(filepath.Join(testCacheRoot, name), cfg.cache)
|
|
|
|
// Cache covers all distros, pass in top directory
|
|
cache := newRPMCache(testCacheRoot, cfg.maxSize)
|
|
|
|
err := cache.shrink()
|
|
assert.NoError(err)
|
|
|
|
// it's hard to predict the exact size after shrink because of directory sizes
|
|
// so let's just check that the new size is between min and max
|
|
assert.LessOrEqual(cfg.minSizeAfterShrink, cache.size)
|
|
assert.Greater(cfg.maxSize, cache.size)
|
|
assert.Equal(len(cfg.repoIDsAfterShrink), len(cache.repoElements))
|
|
for _, id := range cfg.repoIDsAfterShrink {
|
|
assert.Contains(cache.repoElements, id)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Mock package list to use in testing
|
|
var PackageList = rpmmd.PackageList{
|
|
rpmmd.Package{
|
|
Name: "package0",
|
|
Summary: "package summary",
|
|
Description: "package description",
|
|
URL: "https://package-url/",
|
|
Epoch: 0,
|
|
Version: "1.0.0",
|
|
Release: "3",
|
|
Arch: "x86_64",
|
|
License: "MIT",
|
|
},
|
|
}
|
|
|
|
func TestDNFCacheStoreGet(t *testing.T) {
|
|
cache := NewDNFCache(1 * time.Second)
|
|
assert.Equal(t, cache.timeout, 1*time.Second)
|
|
assert.NotNil(t, cache.RWMutex)
|
|
|
|
cache.Store("notreallyahash", PackageList)
|
|
assert.Equal(t, 1, len(cache.results))
|
|
pkgs, ok := cache.Get("notreallyahash")
|
|
assert.True(t, ok)
|
|
assert.Equal(t, "package0", pkgs[0].Name)
|
|
}
|
|
|
|
func TestDNFCacheTimeout(t *testing.T) {
|
|
cache := NewDNFCache(1 * time.Second)
|
|
cache.Store("notreallyahash", PackageList)
|
|
_, ok := cache.Get("notreallyahash")
|
|
assert.True(t, ok)
|
|
time.Sleep(2 * time.Second)
|
|
_, ok = cache.Get("notreallyahash")
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestDNFCacheCleanup(t *testing.T) {
|
|
cache := NewDNFCache(1 * time.Second)
|
|
cache.Store("notreallyahash", PackageList)
|
|
time.Sleep(2 * time.Second)
|
|
assert.Equal(t, 1, len(cache.results))
|
|
cache.CleanCache()
|
|
assert.Equal(t, 0, len(cache.results))
|
|
_, ok := cache.Get("notreallyahash")
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestCleanupOldCacheDirs(t *testing.T) {
|
|
// Run the cleanup without the cache present and with dummy distro names
|
|
CleanupOldCacheDirs("/var/tmp/test-no-cache-rpmmd/", []string{"fedora-37", "fedora-38"})
|
|
|
|
testCacheRoot := t.TempDir()
|
|
// Make all the test caches under root, using their keys as a distro name.
|
|
var distros []string
|
|
for name, cfg := range testCfgs {
|
|
// Cache is now per-distro, use the name of the config as a distro name
|
|
createTestCache(filepath.Join(testCacheRoot, name), cfg)
|
|
distros = append(distros, name)
|
|
}
|
|
sort.Strings(distros)
|
|
|
|
// Add the content of the 'fake-real' cache to the top directory
|
|
// this will be used to simulate an old cache without distro subdirs
|
|
createTestCache(testCacheRoot, testCfgs["fake-real"])
|
|
|
|
CleanupOldCacheDirs(testCacheRoot, distros)
|
|
|
|
// The fake-real files under the root directory should all be gone.
|
|
for path := range testCfgs["fake-real"] {
|
|
_, err := os.Stat(filepath.Join(testCacheRoot, path))
|
|
assert.NotNil(t, err)
|
|
}
|
|
|
|
// The distro cache files should all still be present
|
|
for name, cfg := range testCfgs {
|
|
for path := range cfg {
|
|
_, err := os.Stat(filepath.Join(testCacheRoot, name, path))
|
|
assert.Nil(t, err)
|
|
}
|
|
}
|
|
|
|
// Remove the fake-real distro from the list
|
|
// This simulates retiring an older distribution and cleaning up its cache
|
|
distros = []string{}
|
|
for name := range testCfgs {
|
|
if name == "fake-real" {
|
|
continue
|
|
}
|
|
distros = append(distros, name)
|
|
}
|
|
// Cleanup should now remove the fake-real subdirectory and files
|
|
CleanupOldCacheDirs(testCacheRoot, distros)
|
|
|
|
// The remaining distro's cache files should all still be present
|
|
for _, name := range distros {
|
|
for path := range testCfgs[name] {
|
|
_, err := os.Stat(filepath.Join(testCacheRoot, name, path))
|
|
assert.Nil(t, err)
|
|
}
|
|
}
|
|
|
|
// But the fake-real ones should be gone
|
|
for path := range testCfgs["fake-real"] {
|
|
_, err := os.Stat(filepath.Join(testCacheRoot, "fake-real", path))
|
|
assert.NotNil(t, err)
|
|
}
|
|
}
|