🥊 Resilient testing + log gathering

Convert the bash script to an ansible playbook so we can gracefully
handle testing failures and gather logs reliably. Colorful output
is nice, too.

Signed-off-by: Major Hayden <major@redhat.com>

jenkins/run_tests.sh

@@ -1,44 +1,44 @@
 #!/bin/bash
 set -euxo pipefail
 
-# Ensure Ansible is installed.
-sudo dnf -y install ansible
+# Restart systemd to work around some Fedora issues in cloud images.
+systemctl restart systemd-journald
 
-# Clone the latest version of ansible-osbuild.
-git clone https://github.com/osbuild/ansible-osbuild.git ansible-osbuild
+# Get the current journald cursor.
+export JOURNALD_CURSOR=$(journalctl --quiet -n 1 --show-cursor | tail -n 1 | grep -oP 's\=.*$')
 
-# Get the current SHA of osbuild-composer.
-OSBUILD_COMPOSER_VERSION=$(git rev-parse HEAD)
-
-# Run the deployment.
-pushd ansible-osbuild
-  echo -e "[test_instances]\nlocalhost ansible_connection=local" > hosts.ini
-  ansible-playbook \
-    -i hosts.ini \
-    -e osbuild_composer_repo=${WORKSPACE} \
-    -e osbuild_composer_version=${OSBUILD_COMPOSER_VERSION} \
-    playbook.yml
-popd
-
-run_osbuild_test() {
-  TEST_NAME=$1
-  /usr/libexec/tests/osbuild-composer/${TEST_NAME} \
-    -test.v | tee ${TEST_NAME}.log > /dev/null &
+# Add a function to preserve the system journal if something goes wrong.
+preserve_journal() {
+  journalctl --after-cursor=${JOURNALD_CURSOR} > systemd-journald.log
+  exit 1
 }
+trap "preserve_journal" ERR
 
-# Run the rcm and weldr tests separately to avoid API errors and timeouts.
-run_osbuild_test osbuild-rcm-tests
-run_osbuild_test osbuild-weldr-tests
+# Ensure Ansible is installed.
+if ! rpm -q ansible; then
+  sudo dnf -y install ansible
+fi
 
-# Run the dnf and other tests together.
-TEST_PIDS=()
-run_osbuild_test osbuild-tests
-TEST_PIDS+=($!)
-run_osbuild_test osbuild-dnf-json-tests
-TEST_PIDS+=($!)
-for TEST_PID in "${TEST_PIDS[@]}"; do
-  wait $TEST_PID
-done
+# Write a simple hosts file for Ansible.
+echo -e "[test_instances]\nlocalhost ansible_connection=local" > hosts.ini
 
-# Wait on the image tests for now.
-# /usr/libexec/tests/osbuild-composer/osbuild-image-tests
+# Set Ansible's config file location.
+export ANSIBLE_CONFIG=ansible-osbuild/ansible.cfg
+
+# Deploy the software.
+git clone https://github.com/osbuild/ansible-osbuild.git ansible-osbuild
+ansible-playbook \
+  -i hosts.ini \
+  -e osbuild_composer_repo=${WORKSPACE} \
+  -e osbuild_composer_version=$(git rev-parse HEAD) \
+  ansible-osbuild/playbook.yml
+
+# Run the tests.
+ansible-playbook \
+  -e workspace=${WORKSPACE} \
+  -e journald_cursor="${JOURNALD_CURSOR}" \
+  -i hosts.ini \
+  jenkins/test.yml
+
+# Collect the systemd journal anyway if we made it all the way to the end.
+journalctl --after-cursor=${JOURNALD_CURSOR} > systemd-journald.log

jenkins/test.yml

@@ -0,0 +1,27 @@
+---
+
+- hosts: localhost
+  vars:
+    passed_tests: []
+    failed_tests: []
+    journald_cursor: "{{ ansible_env.JOURNALD_CURSOR"
+  vars_files:
+    - vars.yml
+  tasks:
+
+    - name: Run osbuild-composer tests
+      include_tasks: test_runner.yml
+      loop: "{{ osbuild_composer_base_tests }}"
+      loop_control:
+        loop_var: test
+
+    - name: Show failed and passed tests
+      debug:
+        msg: |
+          Passed tests: "{{ passed_tests }}"
+          Failed tests: "{{ failed_tests }}"
+
+    - name: Fail the test run if a test failed
+      fail:
+        msg: One or more tests failed.
+      when: failed_tests

jenkins/test_runner.yml

@@ -0,0 +1,37 @@
+---
+
+- block:
+
+    # NOTE(mhayden): Running tests from the workspace dirt
+    - name: "Run {{ test.name }} test"
+      command: "{{ tests_path }}/{{ test.name }} -test.v"
+      args:
+        chdir: "{{ tests_working_directory }}"
+      register: async_test
+      async: "{{ test.timeout * 60 }}"
+      poll: "{{ polling_interval }}"
+
+    - name: "Mark {{ test.name }} as passed"
+      set_fact:
+        passed_tests: "{{ passed_tests + [test.name] }}"
+
+  rescue:
+
+    - name: "Mark {{ test.name }} as failed"
+      set_fact:
+        failed_tests: "{{ failed_tests + [test.name] }}"
+
+  always:
+
+    - name: "Write log for {{ test.name }}"
+      copy:
+        dest: "{{ workspace }}/{{ test.name }}.log"
+        content: |
+          Logs from {{ test.name }}
+          ----------------------------------------------------------------------
+          stderr:
+          {{ async_test.stderr }}
+          ----------------------------------------------------------------------
+          stdout:
+          {{ async_test.stdout }}
+

jenkins/vars.yml

@@ -0,0 +1,20 @@
+# Tests should use this as their working directory. This part is critical
+# since dnf-json must be in $PATH.
+tests_working_directory: /usr/libexec/osbuild-composer
+
+# The test executables are here.
+tests_path: /usr/libexec/tests/osbuild-composer
+
+# Set each test executable name and its timeout (in minutes).
+osbuild_composer_base_tests:
+  - name: osbuild-rcm-tests
+    timeout: 5
+  - name: osbuild-weldr-tests
+    timeout: 5
+  - name: osbuild-dnf-json-tests
+    timeout: 15
+  - name: osbuild-tests
+    timeout: 30
+
+# Frequency to check for completed tests.
+polling_interval: 15