🐫 Handle base/image test cases

Speed up test run times by putting base tests and image tests into two
separate jobs in Jenkins. Add support to the testing playbook for both
types of tests and make it easy to add more types later, especially
for image that require booting in the cloud.

Signed-off-by: Major Hayden <major@redhat.com>

jenkins/run_tests.sh

@@ -37,6 +37,7 @@ ansible-playbook \
 ansible-playbook \
   -e workspace=${WORKSPACE} \
   -e journald_cursor="${JOURNALD_CURSOR}" \
+  -e test_type=${TEST_TYPE:-base} \
   -i hosts.ini \
   jenkins/test.yml
 

jenkins/test.yml

@@ -9,11 +9,21 @@
     - vars.yml
   tasks:
 
-    - name: Run osbuild-composer tests
-      include_tasks: test_runner.yml
+    - name: Run osbuild-composer base tests
+      include_tasks: test_runner_base.yml
       loop: "{{ osbuild_composer_base_tests }}"
       loop_control:
         loop_var: test
+      when:
+        - test_type == 'base'
+
+    - name: Run osbuild-composer image tests
+      include_tasks: test_runner_image.yml
+      loop: "{{ osbuild_composer_image_tests }}"
+      loop_control:
+        loop_var: test
+      when:
+        - test_type == 'image'
 
     - name: Show failed and passed tests
       debug:

jenkins/test_runner_base.yml

@@ -10,6 +10,7 @@
       register: async_test
       async: "{{ test.timeout * 60 }}"
       poll: "{{ polling_interval }}"
+      no_log: yes
 
     - name: "Mark {{ test.name }} as passed"
       set_fact:

jenkins/test_runner_image.yml

@@ -0,0 +1,43 @@
+---
+
+- name: Set a fact for the short test case name
+  set_fact:
+    test_case_name: "{{ test.case | splitext | first }}"
+
+- block:
+
+    - name: "Run image test case: {{ test_case_name | basename }}"
+      command: |
+        {{ tests_path }}/{{ test.name }} -test.v \
+          {{ image_test_case_path }}/{{ test.case }}
+      args:
+        chdir: "{{ tests_working_directory }}"
+      register: async_test
+      async: "{{ test.timeout * 60 }}"
+      poll: "{{ polling_interval }}"
+      no_log: yes
+
+    - name: "Mark {{ test_case_name }} as passed"
+      set_fact:
+        passed_tests: "{{ passed_tests + [test_case_name] }}"
+
+  rescue:
+
+    - name: "Mark {{ test_case_name }} as failed"
+      set_fact:
+        failed_tests: "{{ failed_tests + [test_case_name] }}"
+
+  always:
+
+    - name: "Write log for {{ test_case_name }}"
+      copy:
+        dest: "{{ workspace }}/{{ test_case_name }}.log"
+        content: |
+          Logs from image test: {{ test_case_name }}
+          ----------------------------------------------------------------------
+          stderr:
+          {{ async_test.stderr }}
+          ----------------------------------------------------------------------
+          stdout:
+          {{ async_test.stdout }}
+

jenkins/vars.yml

@@ -5,7 +5,7 @@ tests_working_directory: /usr/libexec/osbuild-composer
 # The test executables are here.
 tests_path: /usr/libexec/tests/osbuild-composer
 
-# Set each test executable name and its timeout (in minutes).
+# List of base tests and their timeouts (in minutes)
 osbuild_composer_base_tests:
   - name: osbuild-rcm-tests
     timeout: 5
@@ -16,5 +16,17 @@ osbuild_composer_base_tests:
   - name: osbuild-tests
     timeout: 30
 
+# List of image tests and their timeouts (in minutes).
+osbuild_composer_image_tests:
+  - name: osbuild-image-tests
+    case: fedora_31-x86_64-ext4_filesystem-boot.json
+    timeout: 15
+  - name: osbuild-image-tests
+    case: fedora_31-x86_64-tar-boot.json
+    timeout: 15
+
+# Location of image test case files.
+image_test_case_path: /usr/share/tests/osbuild-composer/cases
+
 # Frequency to check for completed tests.
 polling_interval: 15