0e4a9e586f
Remove all the internal package that are now in the github.com/osbuild/images package and vendor it. A new function in internal/blueprint/ converts from an osbuild-composer blueprint to an images blueprint. This is necessary for keeping the blueprint implementation in both packages. In the future, the images package will change the blueprint (and most likely rename it) and it will only be part of the osbuild-composer internals and interface. The Convert() function will be responsible for converting the blueprint into the new configuration object.
384 lines
16 KiB
Go
384 lines
16 KiB
Go
package dnfjson
|
|
|
|
import (
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/osbuild/images/pkg/rpmmd"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func truncate(path string, size int64) {
|
|
fp, err := os.Create(path)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
defer fp.Close()
|
|
if err := fp.Truncate(size); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
// create a test cache based on the config, where the config keys are file
|
|
// paths and the values are file sizes
|
|
func createTestCache(root string, config testCache) uint64 {
|
|
var totalSize uint64
|
|
for path, fi := range config {
|
|
fullPath := filepath.Join(root, path)
|
|
parPath := filepath.Dir(fullPath)
|
|
if err := os.MkdirAll(parPath, 0770); err != nil {
|
|
panic(err)
|
|
}
|
|
truncate(fullPath, int64(fi.size))
|
|
mtime := time.Unix(fi.mtime, 0)
|
|
if err := os.Chtimes(fullPath, mtime, mtime); err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
// if the path has multiple parts, touch the top level directory of the
|
|
// element
|
|
pathParts := strings.Split(path, "/")
|
|
if len(pathParts) > 1 {
|
|
top := pathParts[0]
|
|
if err := os.Chtimes(filepath.Join(root, top), mtime, mtime); err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
if len(path) >= 64 {
|
|
// paths with shorter names will be ignored by the cache manager
|
|
totalSize += fi.size
|
|
}
|
|
}
|
|
|
|
// add directory sizes to total
|
|
sizer := func(path string, info fs.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if path == root {
|
|
// don't count root
|
|
return nil
|
|
}
|
|
if info.IsDir() {
|
|
totalSize += uint64(info.Size())
|
|
}
|
|
return nil
|
|
}
|
|
if err := filepath.Walk(root, sizer); err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
return totalSize
|
|
}
|
|
|
|
type fileInfo struct {
|
|
size uint64
|
|
mtime int64
|
|
}
|
|
|
|
type testCache map[string]fileInfo
|
|
|
|
var testCfgs = map[string]testCache{
|
|
"rhel84-aarch64": { // real repo metadata file names and sizes
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc.solv": fileInfo{2095095, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-filenames.solvx": fileInfo{14473401, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/gen/groups.xml": fileInfo{1419587, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/3eabd1122210e4def18ae4b96a18aa5bcc186abf2ec14e2e8f1c1bb1ab4d11da-modules.yaml.gz": fileInfo{156314, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/90fd2e7463220a07457e76ae905e1bad754c29e22202bb3202c971a5ece28396-comps-AppStream.aarch64.xml.gz": fileInfo{199426, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/77a66c76b5f6ba51aaee6c0cf76d701601e8b622d1701d1781dabec434f27413-filelists.xml.gz": fileInfo{14370201, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/1941c723c94218eed43eac3174aa94cefbe921e15547c39251a95895024207ca-primary.xml.gz": fileInfo{11439375, 100},
|
|
"9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc-33d346d177279673/repodata/repomd.xml": fileInfo{13285, 100},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62.solv": fileInfo{1147863, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-filenames.solvx": fileInfo{11133964, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-98177081b9162766/repodata/gen/groups.xml": fileInfo{1298102, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-98177081b9162766/repodata/d74783221709ab27d543c1cfc4c02562fde6edfaaaac33ac73a68ecf53188695-comps-BaseOS.aarch64.xml.gz": fileInfo{174076, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-98177081b9162766/repodata/5ded48b4c9e238288130c6670d99f5febdb7273e4a31ac213836a15a2076514d-filelists.xml.gz": fileInfo{11081612, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-98177081b9162766/repodata/8120caf8ebbb8c8b37f6f0dd027d866020ebe7acf9c9ce49ae9903b761986f0c-primary.xml.gz": fileInfo{1836471, 300},
|
|
"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62-98177081b9162766/repodata/repomd.xml": fileInfo{12817, 300},
|
|
},
|
|
"fake-real": { // fake but resembling real data
|
|
"1111111111111111111111111111111111111111111111111111111111111111.solv": fileInfo{100, 0},
|
|
"1111111111111111111111111111111111111111111111111111111111111111-filenames.solv": fileInfo{200, 0},
|
|
"1111111111111111111111111111111111111111111111111111111111111111.whatever": fileInfo{110, 0},
|
|
"1111111111111111111111111111111111111111111111111111111111111111/repodata/a": fileInfo{1000, 0},
|
|
"1111111111111111111111111111111111111111111111111111111111111111/repodata/b": fileInfo{3829, 0},
|
|
"1111111111111111111111111111111111111111111111111111111111111111/repodata/c": fileInfo{831989, 0},
|
|
"2222222222222222222222222222222222222222222222222222222222222222.solv": fileInfo{120, 2},
|
|
"2222222222222222222222222222222222222222222222222222222222222222-filenames.solv": fileInfo{232, 2},
|
|
"2222222222222222222222222222222222222222222222222222222222222222.whatever": fileInfo{110, 2},
|
|
"2222222222222222222222222222222222222222222222222222222222222222/repodata/a": fileInfo{1000, 2},
|
|
"2222222222222222222222222222222222222222222222222222222222222222/repodata/b": fileInfo{3829, 2},
|
|
"2222222222222222222222222222222222222222222222222222222222222222/repodata/c": fileInfo{831989, 2},
|
|
"3333333333333333333333333333333333333333333333333333333333333333.solv": fileInfo{105, 4},
|
|
"3333333333333333333333333333333333333333333333333333333333333333-filenames.solv": fileInfo{200, 4},
|
|
"3333333333333333333333333333333333333333333333333333333333333333.whatever": fileInfo{110, 4},
|
|
"3333333333333333333333333333333333333333333333333333333333333333/repodata/a": fileInfo{2390, 4},
|
|
"3333333333333333333333333333333333333333333333333333333333333333/repodata/b": fileInfo{1234890, 4},
|
|
"3333333333333333333333333333333333333333333333333333333333333333/repodata/c": fileInfo{483, 4},
|
|
},
|
|
"completely-fake": { // just a mess of files (including files without a repo ID)
|
|
"somefile": fileInfo{192, 10291920},
|
|
"yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy-repofiley": fileInfo{29384, 11},
|
|
"yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy-repofiley2": fileInfo{293, 31},
|
|
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-repofile": fileInfo{29384, 30},
|
|
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-repofileb": fileInfo{293, 45},
|
|
},
|
|
}
|
|
|
|
type testCase struct {
|
|
cache testCache
|
|
maxSize uint64
|
|
minSizeAfterShrink uint64
|
|
repoIDsAfterShrink []string
|
|
}
|
|
|
|
func getRepoIDs(ct testCache) []string {
|
|
idMap := make(map[string]bool)
|
|
ids := make([]string, 0)
|
|
for path := range ct {
|
|
if len(path) >= 64 {
|
|
id := path[:64]
|
|
if !idMap[id] {
|
|
idMap[id] = true
|
|
ids = append(ids, id)
|
|
}
|
|
}
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func TestCacheRead(t *testing.T) {
|
|
assert := assert.New(t)
|
|
for name, cfg := range testCfgs {
|
|
t.Run(name, func(t *testing.T) {
|
|
testCacheRoot := t.TempDir()
|
|
// Cache is now per-distro, use the name of the config as a distro name
|
|
s := createTestCache(filepath.Join(testCacheRoot, name), cfg)
|
|
|
|
// Cache covers all distros, pass in top directory
|
|
cache := newRPMCache(testCacheRoot, 1048576) // 1 MiB, but doesn't matter for this test
|
|
|
|
nrepos := len(getRepoIDs(cfg))
|
|
assert.Equal(s, cache.size)
|
|
assert.Equal(nrepos, len(cache.repoElements))
|
|
assert.Equal(nrepos, len(cache.repoRecency))
|
|
})
|
|
}
|
|
}
|
|
|
|
func sizeSum(cfg testCache, repoIDFilter ...string) uint64 {
|
|
var sum uint64
|
|
for path, info := range cfg {
|
|
if len(path) < 64 {
|
|
continue
|
|
}
|
|
rid := path[:64]
|
|
if len(repoIDFilter) == 0 || (len(repoIDFilter) > 0 && strSliceContains(repoIDFilter, rid)) {
|
|
sum += info.size
|
|
}
|
|
}
|
|
return sum
|
|
}
|
|
|
|
func TestCacheCleanup(t *testing.T) {
|
|
rhelRecentRepoSize := sizeSum(testCfgs["rhel84-aarch64"], "df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62")
|
|
rhelTotalRepoSize := sizeSum(testCfgs["rhel84-aarch64"])
|
|
|
|
fakeRealSize2 := sizeSum(testCfgs["fake-real"], "2222222222222222222222222222222222222222222222222222222222222222")
|
|
fakeRealSize3 := sizeSum(testCfgs["fake-real"], "3333333333333333333333333333333333333333333333333333333333333333")
|
|
|
|
testCases := map[string]testCase{
|
|
// max size 1 byte -> clean will delete everything
|
|
"fake-real-full-delete": {
|
|
cache: testCfgs["fake-real"],
|
|
maxSize: 1,
|
|
minSizeAfterShrink: 0,
|
|
},
|
|
"rhel-full-delete": {
|
|
cache: testCfgs["rhel84-aarch64"],
|
|
maxSize: 1,
|
|
minSizeAfterShrink: 0,
|
|
},
|
|
"completely-fake-full-delete": {
|
|
cache: testCfgs["completely-fake"],
|
|
maxSize: 1,
|
|
minSizeAfterShrink: 0,
|
|
},
|
|
"completely-fake-full-delete-2": {
|
|
cache: testCfgs["completely-fake"],
|
|
maxSize: 100,
|
|
minSizeAfterShrink: 0,
|
|
},
|
|
// max size a bit larger than most recent repo -> clean will delete older repos
|
|
"completely-fake-half-delete": {
|
|
cache: testCfgs["completely-fake"],
|
|
maxSize: 29384 + 293 + 1, // one byte larger than the files of one repo
|
|
minSizeAfterShrink: 29384 + 293, // size of files from one repo
|
|
repoIDsAfterShrink: []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, // most recent repo timestamp (45)
|
|
},
|
|
"rhel-half-delete": {
|
|
cache: testCfgs["rhel84-aarch64"],
|
|
maxSize: rhelRecentRepoSize + 102400, // most recent repo file sizes + 100k buffer (for directories)
|
|
minSizeAfterShrink: rhelRecentRepoSize, // after shrink it should be at least as big as the most recent repo
|
|
repoIDsAfterShrink: []string{"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62"}, // most recent repo timestamp (45)
|
|
},
|
|
"fake-real-delete-1": {
|
|
cache: testCfgs["fake-real"],
|
|
maxSize: fakeRealSize3 + fakeRealSize2 + 102400,
|
|
minSizeAfterShrink: fakeRealSize3 + fakeRealSize2,
|
|
repoIDsAfterShrink: []string{"3333333333333333333333333333333333333333333333333333333333333333", "2222222222222222222222222222222222222222222222222222222222222222"},
|
|
},
|
|
"fake-real-delete-2": {
|
|
cache: testCfgs["fake-real"],
|
|
maxSize: fakeRealSize3 + 102400,
|
|
minSizeAfterShrink: fakeRealSize3,
|
|
repoIDsAfterShrink: []string{"3333333333333333333333333333333333333333333333333333333333333333"},
|
|
},
|
|
// max size is huge -> clean wont delete anything
|
|
"rhel-no-delete": {
|
|
cache: testCfgs["rhel84-aarch64"],
|
|
maxSize: 45097156608, // 42 GiB
|
|
minSizeAfterShrink: rhelTotalRepoSize,
|
|
repoIDsAfterShrink: []string{"df2665154150abf76f4d86156228a75c39f3f31a79d4a861d76b1edd89814b62", "9adf133053f0691a0ec12e73cbf1875a90c9268b4f09162fc3387fd76ecb3bcc"},
|
|
},
|
|
}
|
|
|
|
for name, cfg := range testCases {
|
|
t.Run(name, func(t *testing.T) {
|
|
assert := assert.New(t)
|
|
testCacheRoot := t.TempDir()
|
|
// Cache is now per-distro, use the name of the config as a distro name
|
|
createTestCache(filepath.Join(testCacheRoot, name), cfg.cache)
|
|
|
|
// Cache covers all distros, pass in top directory
|
|
cache := newRPMCache(testCacheRoot, cfg.maxSize)
|
|
|
|
err := cache.shrink()
|
|
assert.NoError(err)
|
|
|
|
// it's hard to predict the exact size after shrink because of directory sizes
|
|
// so let's just check that the new size is between min and max
|
|
assert.LessOrEqual(cfg.minSizeAfterShrink, cache.size)
|
|
assert.Greater(cfg.maxSize, cache.size)
|
|
assert.Equal(len(cfg.repoIDsAfterShrink), len(cache.repoElements))
|
|
for _, id := range cfg.repoIDsAfterShrink {
|
|
assert.Contains(cache.repoElements, id)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Mock package list to use in testing
|
|
var PackageList = rpmmd.PackageList{
|
|
rpmmd.Package{
|
|
Name: "package0",
|
|
Summary: "package summary",
|
|
Description: "package description",
|
|
URL: "https://package-url/",
|
|
Epoch: 0,
|
|
Version: "1.0.0",
|
|
Release: "3",
|
|
Arch: "x86_64",
|
|
License: "MIT",
|
|
},
|
|
}
|
|
|
|
func TestDNFCacheStoreGet(t *testing.T) {
|
|
cache := NewDNFCache(1 * time.Second)
|
|
assert.Equal(t, cache.timeout, 1*time.Second)
|
|
assert.NotNil(t, cache.RWMutex)
|
|
|
|
cache.Store("notreallyahash", PackageList)
|
|
assert.Equal(t, 1, len(cache.results))
|
|
pkgs, ok := cache.Get("notreallyahash")
|
|
assert.True(t, ok)
|
|
assert.Equal(t, "package0", pkgs[0].Name)
|
|
}
|
|
|
|
func TestDNFCacheTimeout(t *testing.T) {
|
|
cache := NewDNFCache(1 * time.Second)
|
|
cache.Store("notreallyahash", PackageList)
|
|
_, ok := cache.Get("notreallyahash")
|
|
assert.True(t, ok)
|
|
time.Sleep(2 * time.Second)
|
|
_, ok = cache.Get("notreallyahash")
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestDNFCacheCleanup(t *testing.T) {
|
|
cache := NewDNFCache(1 * time.Second)
|
|
cache.Store("notreallyahash", PackageList)
|
|
time.Sleep(2 * time.Second)
|
|
assert.Equal(t, 1, len(cache.results))
|
|
cache.CleanCache()
|
|
assert.Equal(t, 0, len(cache.results))
|
|
_, ok := cache.Get("notreallyahash")
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestCleanupOldCacheDirs(t *testing.T) {
|
|
// Run the cleanup without the cache present and with dummy distro names
|
|
CleanupOldCacheDirs("/var/tmp/test-no-cache-rpmmd/", []string{"fedora-37", "fedora-38"})
|
|
|
|
testCacheRoot := t.TempDir()
|
|
// Make all the test caches under root, using their keys as a distro name.
|
|
var distros []string
|
|
for name, cfg := range testCfgs {
|
|
// Cache is now per-distro, use the name of the config as a distro name
|
|
createTestCache(filepath.Join(testCacheRoot, name), cfg)
|
|
distros = append(distros, name)
|
|
}
|
|
sort.Strings(distros)
|
|
|
|
// Add the content of the 'fake-real' cache to the top directory
|
|
// this will be used to simulate an old cache without distro subdirs
|
|
createTestCache(testCacheRoot, testCfgs["fake-real"])
|
|
|
|
CleanupOldCacheDirs(testCacheRoot, distros)
|
|
|
|
// The fake-real files under the root directory should all be gone.
|
|
for path := range testCfgs["fake-real"] {
|
|
_, err := os.Stat(filepath.Join(testCacheRoot, path))
|
|
assert.NotNil(t, err)
|
|
}
|
|
|
|
// The distro cache files should all still be present
|
|
for name, cfg := range testCfgs {
|
|
for path := range cfg {
|
|
_, err := os.Stat(filepath.Join(testCacheRoot, name, path))
|
|
assert.Nil(t, err)
|
|
}
|
|
}
|
|
|
|
// Remove the fake-real distro from the list
|
|
// This simulates retiring an older distribution and cleaning up its cache
|
|
distros = []string{}
|
|
for name := range testCfgs {
|
|
if name == "fake-real" {
|
|
continue
|
|
}
|
|
distros = append(distros, name)
|
|
}
|
|
// Cleanup should now remove the fake-real subdirectory and files
|
|
CleanupOldCacheDirs(testCacheRoot, distros)
|
|
|
|
// The remaining distro's cache files should all still be present
|
|
for _, name := range distros {
|
|
for path := range testCfgs[name] {
|
|
_, err := os.Stat(filepath.Join(testCacheRoot, name, path))
|
|
assert.Nil(t, err)
|
|
}
|
|
}
|
|
|
|
// But the fake-real ones should be gone
|
|
for path := range testCfgs["fake-real"] {
|
|
_, err := os.Stat(filepath.Join(testCacheRoot, "fake-real", path))
|
|
assert.NotNil(t, err)
|
|
}
|
|
}
|