diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f3e5d7e3b..0d0748761 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,7 @@
 stages:
   - init
   - rpmbuild
+  - prepare-rhel-internal
   - test
   - finish
 
@@ -26,6 +27,8 @@ init:
 RPM:
   stage: rpmbuild
   extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
   variables:
     EXTRA_REPO_PATH_SEGMENT: "gitlab/"
   script:
@@ -52,6 +55,8 @@ RPM:
 Container:
   stage: rpmbuild
   extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
   variables:
     EXTRA_REPO_PATH_SEGMENT: "gitlab/"
   script:
@@ -61,9 +66,33 @@ Container:
       - RUNNER:
           - aws/fedora-33-x86_64
 
+Prepare-rhel-internal:
+  stage: prepare-rhel-internal
+  extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+  variables:
+    EXTRA_REPO_PATH_SEGMENT: "gitlab/"
+  script:
+    - schutzbot/prepare-rhel-internal.sh
+  artifacts:
+    paths:
+      - rhel-8.json
+      - rhel8internal.repo
+      - rhel-8-beta.json
+      - COMPOSE_ID
+  parallel:
+    matrix:
+      - RUNNER:
+          - aws/rhel-8.4-x86_64
+        INTERNAL_NETWORK: ["true"]
+
 Base:
   stage: test
   extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-[8-9]\.[\S]+/'
   variables:
     EXTRA_REPO_PATH_SEGMENT: "gitlab/"
   script:
@@ -96,6 +125,9 @@ Base:
 OSTree:
   stage: test
   extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-[8-9]\.[\S]+/'
   variables:
     EXTRA_REPO_PATH_SEGMENT: "gitlab/"
   script:
@@ -135,6 +167,9 @@ New OSTree:
 Integration:
   stage: test
   extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-[8-9]\.[\S]+/'
   variables:
     EXTRA_REPO_PATH_SEGMENT: "gitlab/"
   script:
@@ -164,7 +199,6 @@ Integration:
         INTERNAL_NETWORK: ["true"]
         DISTRO_CODE: ["rhel_90"]
 
-
 .API_TESTS: &API_TESTS
   TARGET:
     - azure
@@ -175,6 +209,9 @@ Integration:
 API:
   stage: test
   extends: .terraform
+  rules:
+    - if: '$CI_PIPELINE_SOURCE != "schedule"'
+    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-[8-9]\.[\S]+/'
   variables:
     EXTRA_REPO_PATH_SEGMENT: "gitlab/"
   script:
diff --git a/schutzbot/deploy.sh b/schutzbot/deploy.sh
index d60175540..e46189e62 100755
--- a/schutzbot/deploy.sh
+++ b/schutzbot/deploy.sh
@@ -100,7 +100,8 @@ fi
 if [ -f "rhel8internal.repo" ]; then
     greenprint "Preparing repos for internal build testing"
     sudo mv rhel8internal.repo /etc/yum.repos.d/
-    sudo rm -f /etc/yum.repos.d/osbuild*.repo
+    # Change back to removing osbuild*.repo when we have rpms in 8.5
+    sudo rm -f /etc/yum.repos.d/osbuild-composer.repo
 fi
 
 greenprint "Installing test packages for ${PROJECT}"
diff --git a/schutzbot/prepare-rhel-internal.sh b/schutzbot/prepare-rhel-internal.sh
index e286c49b5..4b7d8fdb9 100755
--- a/schutzbot/prepare-rhel-internal.sh
+++ b/schutzbot/prepare-rhel-internal.sh
@@ -8,7 +8,12 @@ function greenprint {
 
 ARCH=$(uname -m)
 
-source ../tools/define-compose-url.sh
+if [ -e ../tools/define-compose-url.sh ]
+then
+    source ../tools/define-compose-url.sh
+else
+    source ./tools/define-compose-url.sh
+fi
 
 # Create a repository file for installing the osbuild-composer RPMs
 greenprint "📜 Generating dnf repository file"
@@ -67,6 +72,7 @@ if ! s3cmd --version > /dev/null 2>&1; then
     sudo pip3 -q install s3cmd
 fi
 
+JOB_NAME="${JOB_NAME:-${CI_JOB_ID}}"
 REPO_DIR_LATEST="repo/${JOB_NAME}/latest/internal"
 mkdir -p "$REPO_DIR_LATEST"
 
diff --git a/tools/define-compose-url.sh b/tools/define-compose-url.sh
index 686172617..8ac8c0cf8 100755
--- a/tools/define-compose-url.sh
+++ b/tools/define-compose-url.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euo pipefail
 
-curl -L http://download.devel.redhat.com/rhel-8/nightly/RHEL-8/latest-RHEL-8.4/COMPOSE_ID > COMPOSE_ID
+curl -L http://download.devel.redhat.com/rhel-8/nightly/RHEL-8/latest-finished-RHEL-8.5/COMPOSE_ID > COMPOSE_ID
 COMPOSE_ID=$(cat COMPOSE_ID)
 
 # default to a nightly tree but respect values passed from ENV so we can test rel-eng composes as well
@@ -10,3 +10,11 @@ COMPOSE_URL="${COMPOSE_URL:-http://download.devel.redhat.com/rhel-8/nightly/RHEL
 # in case COMPOSE_URL was defined from the outside refresh COMPOSE_ID file,
 # used for telegram messages in case of success/failure
 curl -L "$COMPOSE_URL/COMPOSE_ID" > COMPOSE_ID
+
+# Make sure the compose URL really exists
+RETURN_CODE=$(curl --silent -o -I -L -s -w "%{http_code}" "${COMPOSE_URL}")
+if [ "$RETURN_CODE" != 200 ]
+then
+    echo "Compose URL $COMPOSE_URL returned error code $RETURN_CODE, exiting."
+    exit 1
+fi
diff --git a/tools/provision.sh b/tools/provision.sh
index cfc35a451..bf1d4f60e 100755
--- a/tools/provision.sh
+++ b/tools/provision.sh
@@ -63,11 +63,11 @@ esac
 
 # overrides for RHEL nightly builds testing
 if [ -f "rhel-8.json" ]; then
-    sudo mv rhel-8.json /etc/osbuild-composer/repositories/
+    sudo cp rhel-8.json /etc/osbuild-composer/repositories/
 fi
 
 if [ -f "rhel-8-beta.json" ]; then
-    sudo mv rhel-8-beta.json /etc/osbuild-composer/repositories/
+    sudo cp rhel-8-beta.json /etc/osbuild-composer/repositories/
 fi
 
 # Generate all X.509 certificates for the tests