test: separate regression from base tests

- Rewrote regression.sh to be like base tests with list of regression
  test scripts to be run and fail counts
- Separate regression tests in CI to have better control of distro
  runners

.gitlab-ci.yml

@@ -95,7 +95,6 @@ Base:
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/base_tests.sh
-    - /usr/libexec/tests/osbuild-composer/regression.sh
   parallel:
     matrix:
       - RUNNER:
@@ -119,6 +118,31 @@ Base:
       - "*.repo"
     when: always
 
+Regression:
+  stage: test
+  extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-[8-9]\.[\S]+/'
+  script:
+    - schutzbot/deploy.sh
+    - /usr/libexec/tests/osbuild-composer/regression.sh
+  parallel:
+    matrix:
+      - RUNNER:
+          - aws/rhel-8-x86_64
+          - aws/rhel-8-aarch64
+          - aws/centos-stream-8-x86_64
+          - aws/centos-stream-8-aarch64
+      - RUNNER:
+          - aws/rhel-8.5-x86_64
+        INTERNAL_NETWORK: ["true"]
+  artifacts:
+    paths:
+      - journal-log
+      - "*.repo"
+    when: always
+
 OSTree:
   stage: test
   extends: .terraform

test/cases/regression.sh

@@ -1,20 +1,61 @@
 #!/bin/bash
 set -euo pipefail
 
-# Get OS data.
+TESTS_PATH=/usr/libexec/tests/osbuild-composer/
-source /etc/os-release
+mkdir --parents /tmp/logs
+LOGS_DIRECTORY=$(mktemp --directory --tmpdir=/tmp/logs)
+
+PASSED_TESTS=()
+FAILED_TESTS=()
+
+TEST_CASES=(
+    "regression-excluded-dependency.sh"
+    "regression-include-excluded-packages.sh"
+)
+
+# Print out a nice test divider so we know when tests stop and start.
+test_divider () {
+    printf "%0.s-" {1..78} && echo
+}
+
+# Run a test case and store the result as passed or failed.
+run_test_case () {
+    TEST_NAME=$(basename "$1")
+    echo
+    test_divider
+    echo "🏃🏻 Running test: ${TEST_NAME}"
+    test_divider
+
+    if sudo "${1}" -test.v | tee "${LOGS_DIRECTORY}"/"${TEST_NAME}".log; then
+        PASSED_TESTS+=("$TEST_NAME")
+    else
+        FAILED_TESTS+=("$TEST_NAME")
+    fi
+
+    test_divider
+    echo
+}
+
 
 # Provision the software under tet.
 /usr/libexec/osbuild-composer-test/provision.sh
 
-# Set os-variant and boot location used by virt-install.
+# Run each test case.
-case "${ID}" in
+for TEST_CASE in "${TEST_CASES[@]}"; do
-    "fedora")
+    run_test_case ${TESTS_PATH}/"$TEST_CASE"
-        echo "No regression test for Fedora";;
+done
-    "rhel")
+
-        /usr/libexec/tests/osbuild-composer/regression-include-excluded-packages.sh;;
+# Print a report of the test results.
-    "centos")
+test_divider
-        /usr/libexec/tests/osbuild-composer/regression-include-excluded-packages.sh;;
+echo "😃 Passed tests:" "${PASSED_TESTS[@]}"
-    *)
+echo "☹ Failed tests:" "${FAILED_TESTS[@]}"
-        echo "unsupported distro: ${ID}-${VERSION_ID}"
+test_divider
-esac
+
+# Exit with a failure if tests were executed and any of them failed.
+if [ ${#PASSED_TESTS[@]} -gt 0 ] && [ ${#FAILED_TESTS[@]} -eq 0 ]; then
+    echo "🎉 All tests passed."
+    exit 0
+else
+    echo "🔥 One or more tests failed."
+    exit 1
+fi