diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index a3dea260..764d6211 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -7,9 +7,6 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  # setting workers to 1 just to have the same syntax in all test
-  # but to disable parallelism for the majority who have concurrency issues
-  TEST_WORKERS: "-n 1"
   # Share the store between the workers speeds things up further
   OSBUILD_TEST_STORE: /var/tmp/osbuild-test-store
 
@@ -21,26 +18,12 @@ jobs:
       fail-fast: false
       matrix:
         test:
-          - "test.mod"
-          - "test.run.test_boot"
-          - "test.run.test_devices"
-          - "test.run.test_executable"
-          - "test.run.test_mount"
-          - "test.run.test_noop"
-          - "test.run.test_sources"
-          - "test.run.test_stages"
-          - "stages/test"
-          - "tools/test"
+          - parallel
+          - normal
         environment:
           - "py36"    # RH8
           - "py39"    # RH9
           - "py312"   # latest fedora
-        # special keyword "include"
-        include:
-          # this test _can_ run in parallel
-          # Using 4 workers is a bit arbitrary, "auto" is probably too aggressive.
-          - test: "test.run.test_stages"
-            env_TEST_WORKERS: "-n 4"
     steps:
     - name: "Clone Repository"
       uses: actions/checkout@v4
@@ -54,9 +37,16 @@ jobs:
           # it to vfs for the local storage skopeo stage test.
           sed -i 's/overlay/vfs/g' /usr/share/containers/storage.conf  # default system config
           sed -i 's/overlay/vfs/g' /etc/containers/storage.conf || true  # potential overrides
-          TEST_WORKERS="${{ matrix.env_TEST_WORKERS || env.TEST_WORKERS }}" \
+          if [ "${{ matrix.test }}" == "parallel" ]; then
+              # 4 is a bit arbitrary
+              TEST_WORKERS="-n 4"
+              TEST_CATEGORY="test_stages.py"
+          else
+              # test_assemblers.py is run below
+              TEST_CATEGORY="not test_stages.py and not test_assemblers.py"
+          fi
           OSBUILD_TEST_STORE="${{ env.OSBUILD_TEST_STORE }}" \
-          tox -e "${{ matrix.environment }}" -- ${{ matrix.test }}
+          tox -e "${{ matrix.environment }}" -- $TEST_WORKERS -k "$TEST_CATEGORY"
 
   v1_manifests:
     name: "Assembler test (legacy)"
@@ -72,6 +62,5 @@ jobs:
         with:
           image: ghcr.io/osbuild/osbuild-ci:latest-202308241910
           run: |
-            TEST_WORKERS="${{ env.TEST_WORKERS }}" \
             OSBUILD_TEST_STORE="${{ env.OSBUILD_TEST_STORE }}" \
-            tox -e "py36" -- test.run.test_assemblers
+            tox -e "py36" -- ${{ env.TEST_WORKERS }} test.run.test_assemblers