From 2444e633fb1f9dc4473575bf39b4c1df3ac11999 Mon Sep 17 00:00:00 2001
From: Achilleas Koutsou <achilleas@koutsou.net>
Date: Tue, 13 Jul 2021 14:18:14 +0200
Subject: [PATCH] test: separate regression from base tests

- Rewrote regression.sh to be like base tests with list of regression
  test scripts to be run and fail counts
- Separate regression tests in CI to have better control of distro
  runners
---
 .gitlab-ci.yml           | 26 +++++++++++++++-
 test/cases/regression.sh | 67 ++++++++++++++++++++++++++++++++--------
 2 files changed, 79 insertions(+), 14 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 25f0b886e..9e84b2c0e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -95,7 +95,6 @@ Base:
   script:
     - schutzbot/deploy.sh
     - /usr/libexec/tests/osbuild-composer/base_tests.sh
-    - /usr/libexec/tests/osbuild-composer/regression.sh
   parallel:
     matrix:
       - RUNNER:
@@ -119,6 +118,31 @@ Base:
       - "*.repo"
     when: always
 
+Regression:
+  stage: test
+  extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-[8-9]\.[\S]+/'
+  script:
+    - schutzbot/deploy.sh
+    - /usr/libexec/tests/osbuild-composer/regression.sh
+  parallel:
+    matrix:
+      - RUNNER:
+          - aws/rhel-8-x86_64
+          - aws/rhel-8-aarch64
+          - aws/centos-stream-8-x86_64
+          - aws/centos-stream-8-aarch64
+      - RUNNER:
+          - aws/rhel-8.5-x86_64
+        INTERNAL_NETWORK: ["true"]
+  artifacts:
+    paths:
+      - journal-log
+      - "*.repo"
+    when: always
+
 OSTree:
   stage: test
   extends: .terraform
diff --git a/test/cases/regression.sh b/test/cases/regression.sh
index 45979c9c9..2d3b860a4 100644
--- a/test/cases/regression.sh
+++ b/test/cases/regression.sh
@@ -1,20 +1,61 @@
 #!/bin/bash
 set -euo pipefail
 
-# Get OS data.
-source /etc/os-release
+TESTS_PATH=/usr/libexec/tests/osbuild-composer/
+mkdir --parents /tmp/logs
+LOGS_DIRECTORY=$(mktemp --directory --tmpdir=/tmp/logs)
+
+PASSED_TESTS=()
+FAILED_TESTS=()
+
+TEST_CASES=(
+    "regression-excluded-dependency.sh"
+    "regression-include-excluded-packages.sh"
+)
+
+# Print out a nice test divider so we know when tests stop and start.
+test_divider () {
+    printf "%0.s-" {1..78} && echo
+}
+
+# Run a test case and store the result as passed or failed.
+run_test_case () {
+    TEST_NAME=$(basename "$1")
+    echo
+    test_divider
+    echo "🏃🏻 Running test: ${TEST_NAME}"
+    test_divider
+
+    if sudo "${1}" -test.v | tee "${LOGS_DIRECTORY}"/"${TEST_NAME}".log; then
+        PASSED_TESTS+=("$TEST_NAME")
+    else
+        FAILED_TESTS+=("$TEST_NAME")
+    fi
+
+    test_divider
+    echo
+}
+
 
 # Provision the software under tet.
 /usr/libexec/osbuild-composer-test/provision.sh
 
-# Set os-variant and boot location used by virt-install.
-case "${ID}" in
-    "fedora")
-        echo "No regression test for Fedora";;
-    "rhel")
-        /usr/libexec/tests/osbuild-composer/regression-include-excluded-packages.sh;;
-    "centos")
-        /usr/libexec/tests/osbuild-composer/regression-include-excluded-packages.sh;;
-    *)
-        echo "unsupported distro: ${ID}-${VERSION_ID}"
-esac
+# Run each test case.
+for TEST_CASE in "${TEST_CASES[@]}"; do
+    run_test_case ${TESTS_PATH}/"$TEST_CASE"
+done
+
+# Print a report of the test results.
+test_divider
+echo "😃 Passed tests:" "${PASSED_TESTS[@]}"
+echo "☹ Failed tests:" "${FAILED_TESTS[@]}"
+test_divider
+
+# Exit with a failure if tests were executed and any of them failed.
+if [ ${#PASSED_TESTS[@]} -gt 0 ] && [ ${#FAILED_TESTS[@]} -eq 0 ]; then
+    echo "🎉 All tests passed."
+    exit 0
+else
+    echo "🔥 One or more tests failed."
+    exit 1
+fi