fscache: add new FsCache._last_used() helper

This helper can be used to implement a strategy to find the oldest
cache entries and evict them when the cache is full.

The implementation uses the `atime` of the per object `cache.lock`
file and ensures in `load()` that it's actually updated.
This commit is contained in:
Michael Vogt 2023-12-12 13:21:54 +01:00 committed by Simon de Vlieger
parent f52cabc3c1
commit caddf0adfb
2 changed files with 92 additions and 7 deletions

View file

@ -4,10 +4,14 @@
# pylint: disable=protected-access
import contextlib
import json
import os
import pathlib
import subprocess
import sys
import tempfile
import time
import pytest
@ -20,6 +24,22 @@ def tmpdir_fixture():
yield tmp
def sleep_for_fs():
"""Sleep a tiny amount of time for atime/mtime updates to show up in fs"""
time.sleep(0.05)
def has_precise_fs_timestamps():
with tempfile.TemporaryDirectory(dir="/var/tmp") as tmpdir:
stamp_path = pathlib.Path(tmpdir) / "stamp"
stamp_path.write_bytes(b"m1")
mtime1 = stamp_path.stat().st_mtime
sleep_for_fs()
stamp_path.write_bytes(b"m2")
mtime2 = stamp_path.stat().st_mtime
return mtime2 > mtime1
def test_calculate_space(tmpdir):
#
# Test the `_calculate_space()` helper and verify it only includes file
@ -398,6 +418,51 @@ def test_size_discard(tmpdir):
with cache.load("foo") as rpath:
pass
def test_cache_last_used_noent(tmpdir):
cache = fscache.FsCache("osbuild-test-appid", tmpdir)
with pytest.raises(fscache.FsCache.MissError):
cache._last_used("non-existant-entry")
@pytest.mark.skipif(not has_precise_fs_timestamps(), reason="need precise fs timestamps")
def test_cache_load_updates_last_used(tmpdir):
cache = fscache.FsCache("osbuild-test-appid", tmpdir)
with cache:
cache.info = cache.info._replace(maximum_size=1024*1024)
with cache.store("foo") as rpath:
pass
with cache.load("foo") as rpath:
pass
load_time1 = cache._last_used("foo")
# would be nice to have a helper for this in cache
obj_lock_path = os.path.join(
cache._dirname_objects, "foo", cache._filename_object_lock)
mtime1 = os.stat(cache._path(obj_lock_path)).st_mtime
assert load_time1 > 0
sleep_for_fs()
with cache.load("foo") as rpath:
pass
# load time is updated
load_time2 = cache._last_used("foo")
assert load_time2 > load_time1
# mtime is unchanged
mtime2 = os.stat(cache._path(obj_lock_path)).st_mtime
assert mtime1 == mtime2
@pytest.mark.skipif(os.getuid() != 0, reason="needs root")
def test_cache_load_updates_last_used_on_noatime(tmp_path):
mnt_path = tmp_path / "mnt"
mnt_path.mkdir()
with contextlib.ExitStack() as cm:
subprocess.check_call(
["mount", "-t", "tmpfs", "-o", "noatime", "none", os.fspath(mnt_path)],
stdout=sys.stdout,
stderr=sys.stderr,
)
cm.callback(subprocess.check_call, ["umount", os.fspath(mnt_path)], stdout=sys.stdout, stderr=sys.stderr)
test_cache_load_updates_last_used(mnt_path)
def test_cache_full_behavior(tmp_path):
cache = fscache.FsCache("osbuild-cache-evict", tmp_path)
@ -422,7 +487,7 @@ def test_cache_full_behavior(tmp_path):
assert cache._calculate_space(tmp_path) < 192 * 1024
with cache.load("o2") as o:
assert o != ""
# adding a third one will (silently) fail
# adding a third one will (silently) fail because the cache is full
with cache.store("o3") as rpath:
rpath_f3 = os.path.join(tmp_path, rpath, "f3")
with open(rpath_f3, "wb") as fp: