From 4aeb6f2b28c1c559493899ecb0eed5d2fd498bf0 Mon Sep 17 00:00:00 2001
From: Major Hayden <major@redhat.com>
Date: Wed, 13 May 2020 16:02:49 -0500
Subject: [PATCH] CI: Split image tests into separate tasks

Make it easier to see progress in Jenkins for image tests and keep their
logs separated for easier diagnosis of problems.

Signed-off-by: Major Hayden <major@redhat.com>
---
 schutzbot/test.yml              |  3 +++
 schutzbot/test_runner_image.yml | 36 +++++++++------------------------
 2 files changed, 13 insertions(+), 26 deletions(-)

diff --git a/schutzbot/test.yml b/schutzbot/test.yml
index e0359943b..a1c7ee98c 100644
--- a/schutzbot/test.yml
+++ b/schutzbot/test.yml
@@ -19,6 +19,9 @@
 
     - name: Run osbuild-composer image tests
       include_tasks: test_runner_image.yml
+      loop: "{{ osbuild_composer_image_test_cases }}"
+      loop_control:
+        loop_var: test_case
       when:
         - test_type == 'image'
 
diff --git a/schutzbot/test_runner_image.yml b/schutzbot/test_runner_image.yml
index a22582307..876b5c3d8 100644
--- a/schutzbot/test_runner_image.yml
+++ b/schutzbot/test_runner_image.yml
@@ -9,50 +9,34 @@
       rhel_{{ ansible_distribution_version }}-{{ ansible_machine }}
       {%- endif -%}
 
-- name: Show which tests will be run
-  debug:
-    msg: |
-      Running the following image test cases:
-      {% for test_case in osbuild_composer_image_test_cases %}
-        - {{ test_case | splitext | first }}
-      {% endfor %}
-
 - block:
 
-    # NOTE(mhayden): The fancy jinja2 here ensures that the last test case
-    # in the list does not have a backslash added. That would make the shell
-    # upset and nobody likes it when that happens.
-    - name: "Run image tests"
+    - name: "Run image test case: {{ test_case_prefix }}-{{ test_case }}"
       command: |
-        {{ image_test_executable }} -test.v -test.timeout {{image_test_timeout * 60}}s \
-          {% for test_case in osbuild_composer_image_test_cases %}
-            {% if loop.last %}
+        {{ image_test_executable }} -test.v \
+          -test.timeout {{image_test_timeout * 60}}s \
           {{ image_test_case_path }}/{{ test_case_prefix }}-{{ test_case }}
-            {% else %}
-          {{ image_test_case_path }}/{{ test_case_prefix }}-{{ test_case }} \
-            {% endif %}
-          {% endfor %}
       args:
         chdir: "{{ tests_working_directory }}"
       register: test_cmd
 
-    - name: "Mark image tests as passed"
+    - name: "Mark test case as passed: {{ test_case_prefix }}-{{ test_case }}"
       set_fact:
-        passed_tests: "{{ passed_tests + ['image_tests'] }}"
+        passed_tests: "{{ passed_tests + [ test_case_prefix + '-' + test_case ] }}"
 
   rescue:
 
-    - name: "Mark image tests as failed"
+    - name: "Mark test case as failed: {{ test_case_prefix }}-{{ test_case }}"
       set_fact:
-        failed_tests: "{{ failed_tests + ['image_tests'] }}"
+        failed_tests: "{{ failed_tests + [ test_case_prefix + '-' + test_case ] }}"
 
   always:
 
-    - name: "Write log for image tests"
+    - name: "Write log for test case: {{ test_case_prefix }}-{{ test_case }}"
       copy:
-        dest: "{{ workspace }}/image_test.log"
+        dest: "{{ workspace }}/{{ test_case_prefix }}-{{ test_case }}.log"
         content: |
-          Logs from image test
+          Logs from test case: {{ test_case_prefix }}-{{ test_case }}
           ----------------------------------------------------------------------
           stderr:
           {{ test_cmd.stderr }}