From 9a47a56639bfbe47dbd51b4f01db8d4cb94bc80e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hozza?= <thozza@redhat.com>
Date: Mon, 29 May 2023 14:00:54 +0200
Subject: [PATCH] mockbuild.sh: retry dnf install up to 5 times
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We have been observing a race condition in our CI when installing
packages at the beginning of the SUT setup. This happens only on RHEL
and it is caused by the `rhc` tool, which executes some Ansible
playbooks on system startup, which install packages using dnf. This
interferes with dnf commands ran by the `mockbuild.sh` script, which
results in a job failure.

Since there seems to be no way to determine if `rhc` finished "its
thing", let's retry dnf install of packages up to 5 times with
exponential backoff in between retries.

Signed-off-by: Tomáš Hozza <thozza@redhat.com>
---
 schutzbot/mockbuild.sh | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh
index 7a3b3135..8d49444b 100755
--- a/schutzbot/mockbuild.sh
+++ b/schutzbot/mockbuild.sh
@@ -32,6 +32,27 @@ function template_override {
     echo '"""' | sudo tee -a /etc/mock/templates/"$TEMPLATE"
 }
 
+# Retry dnf install up to 5 times with exponential backoff time
+function dnf_install_with_retry {
+    local n=1
+    local attempts=5
+    local timeout=1
+    while true; do
+        if sudo dnf install -y "$@"; then
+            break
+        elif [ $n -lt $attempts ]; then
+            ((n++))
+            # exponentially increase the timeout
+            timeout=$((n ** 2))
+            echo "Retrying dnf install in $timeout seconds..."
+            sleep "$timeout"
+        else
+            echo "dnf install failed after $n attempts: aborting" >&2
+            return 1
+        fi
+    done
+}
+
 # Get OS and architecture details.
 source tools/set-env-variables.sh
 
@@ -90,12 +111,12 @@ if [[ $ID == rhel || $ID == centos ]] && ! rpm -q epel-release; then
     greenprint "📦 Setting up EPEL repository"
     curl -Ls --retry 5 --output /tmp/epel.rpm \
         https://dl.fedoraproject.org/pub/epel/epel-release-latest-${VERSION_ID%.*}.noarch.rpm
-    sudo dnf install -y /tmp/epel.rpm
+    dnf_install_with_retry /tmp/epel.rpm
 fi
 
 # Install requirements for building RPMs in mock.
 greenprint "📦 Installing mock requirements"
-sudo dnf -y install createrepo_c make mock python3-pip rpm-build s3cmd
+dnf_install_with_retry createrepo_c make mock python3-pip rpm-build s3cmd
 
 # Print some data.
 greenprint "🧬 Using mock config: ${MOCK_CONFIG}"