diff --git a/tools/test-case-generators/generate-test-cases b/tools/test-case-generators/generate-test-cases
index 462e91305..01dc4a4e0 100755
--- a/tools/test-case-generators/generate-test-cases
+++ b/tools/test-case-generators/generate-test-cases
@@ -63,8 +63,13 @@ class TestCaseGenerator:
     def __init__(self, test_case_request):
         self.test_case = test_case_request
 
-    def get_test_case(self, no_image_info, store):
-        compose_request = json.dumps(self.test_case["compose-request"])
+    def ensure_dnf_json(self):
+        if os.path.exists("/.dockerenv"):
+            if os.path.exists("/run/osbuild-dnf-json/api.sock"):
+                # assume we are good to go
+                return
+            print("running in container and no dnf-json socket found")
+            sys.exit(1)
 
         command = ["systemctl", "is-active", "--quiet", "osbuild-dnf-json.socket"]
         if not is_subprocess_succeeding(command):
@@ -74,6 +79,11 @@ class TestCaseGenerator:
                 print("impossible to start osbuild-dnf-json.socket with systemd")
                 sys.exit(1)
 
+    def get_test_case(self, no_image_info, store):
+        compose_request = json.dumps(self.test_case["compose-request"])
+
+        self.ensure_dnf_json()
+
         pipeline_command = ["go", "run", "./cmd/osbuild-pipeline", "-"]
         self.test_case["manifest"] = json.loads(get_subprocess_stdout(pipeline_command, input=compose_request, encoding="utf-8"))