diff --git a/osbuild/testutil/__init__.py b/osbuild/testutil/__init__.py
index c008ab13..afd304cb 100644
--- a/osbuild/testutil/__init__.py
+++ b/osbuild/testutil/__init__.py
@@ -80,6 +80,7 @@ def mock_command(cmd_name: str, script: str):
             os.environ["PATH"] = original_path
 
 
+@contextlib.contextmanager
 def make_container(tmp_path, tag, fake_content, base="scratch"):
     fake_container_src = tmp_path / "fake-container-src"
     make_fake_tree(fake_container_src, fake_content)
@@ -95,3 +96,7 @@ def make_container(tmp_path, tag, fake_content, base="scratch"):
         "-f", os.fspath(fake_containerfile_path),
         "-t", tag,
     ])
+    try:
+        yield
+    finally:
+        subprocess.check_call(["podman", "image", "rm", tag])
diff --git a/sources/test/test_container_storage_source.py b/sources/test/test_container_storage_source.py
index 13920781..b10316f7 100644
--- a/sources/test/test_container_storage_source.py
+++ b/sources/test/test_container_storage_source.py
@@ -17,12 +17,12 @@ SOURCES_NAME = "org.osbuild.containers-storage"
 @pytest.mark.skipif(os.getuid() != 0, reason="root only")
 def test_containers_storage_integration(tmp_path, sources_module):
     base_tag = "container-" + "".join(random.choices(string.digits, k=12))
-    make_container(tmp_path, base_tag, {
+    with make_container(tmp_path, base_tag, {
         "file1": "file1 content",
-    })
-    image_id = subprocess.check_output(["podman", "inspect", "-f", "{{ .Id }}", base_tag],
-                                       universal_newlines=True).strip()
-    checksum = f"sha256:{image_id}"
-    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-    cnt_storage = sources_module.ContainersStorageSource.from_args(["--service-fd", str(sock.fileno())])
-    assert cnt_storage.exists(checksum, None)
+    }):
+        image_id = subprocess.check_output(["podman", "inspect", "-f", "{{ .Id }}", base_tag],
+                                           universal_newlines=True).strip()
+        checksum = f"sha256:{image_id}"
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        cnt_storage = sources_module.ContainersStorageSource.from_args(["--service-fd", str(sock.fileno())])
+        assert cnt_storage.exists(checksum, None)
diff --git a/stages/test/test_container_deploy.py b/stages/test/test_container_deploy.py
index 67cdf42b..fb18c438 100644
--- a/stages/test/test_container_deploy.py
+++ b/stages/test/test_container_deploy.py
@@ -11,7 +11,7 @@ from unittest.mock import call, patch
 import pytest
 
 import osbuild.testutil
-from osbuild.testutil import has_executable, make_container, make_fake_tree
+from osbuild.testutil import has_executable, make_container
 
 STAGE_NAME = "org.osbuild.container-deploy"
 
@@ -22,19 +22,17 @@ def test_container_deploy_integration(tmp_path, stage_module):
     # build two containers and overlay files to test for
     # https://github.com/containers/storage/issues/1779
     base_tag = "cont-base-" + "".join(random.choices(string.digits, k=12))
-    make_container(tmp_path, base_tag, {"file1": "file1 from base"})
-    cont_tag = "cont" + "".join(random.choices(string.digits, k=12))
-    make_container(tmp_path, cont_tag, {"file1": "file1 from final layer"}, base_tag)
-    # export for the container-deploy stage
-    fake_container_dst = tmp_path / "fake-container"
-    subprocess.check_call([
-        "podman", "save",
-        "--format=oci-archive",
-        f"--output={fake_container_dst}",
-        cont_tag,
-    ])
-    # and remove from podman
-    subprocess.check_call(["podman", "rmi", cont_tag, base_tag])
+    with make_container(tmp_path, base_tag, {"file1": "file1 from base"}):
+        cont_tag = "cont" + "".join(random.choices(string.digits, k=12))
+        with make_container(tmp_path, cont_tag, {"file1": "file1 from final layer"}, base_tag):
+            # export for the container-deploy stage
+            fake_container_dst = tmp_path / "fake-container"
+            subprocess.check_call([
+                "podman", "save",
+                "--format=oci-archive",
+                f"--output={fake_container_dst}",
+                cont_tag,
+            ])
 
     inputs = {
         "images": {
@@ -66,22 +64,20 @@ def test_container_deploy_integration(tmp_path, stage_module):
 @pytest.mark.skipif(not has_executable("podman"), reason="no podman executable")
 def test_container_deploy_exclude(tmp_path, stage_module):
     base_tag = "cont-base-" + "".join(random.choices(string.digits, k=12))
-    make_container(tmp_path, base_tag, {
+    with make_container(tmp_path, base_tag, {
         "file1": "file1 content",
         "file2": "file2 content",
         "dir1/file3": "dir1/file3 content",
         "dir2/file4": "dir2/file4 content",
-    })
-    # export for the container-deploy stage
-    fake_container_dst = tmp_path / "fake-container"
-    subprocess.check_call([
-        "podman", "save",
-        "--format=oci-archive",
-        f"--output={fake_container_dst}",
-        base_tag,
-    ])
-    # and remove from podman
-    subprocess.check_call(["podman", "rmi", base_tag])
+    }):
+        # export for the container-deploy stage
+        fake_container_dst = tmp_path / "fake-container"
+        subprocess.check_call([
+            "podman", "save",
+            "--format=oci-archive",
+            f"--output={fake_container_dst}",
+            base_tag,
+        ])
 
     inputs = {
         "images": {