From 381abb354a3ada7b08b141bdc2cf987f887b877f Mon Sep 17 00:00:00 2001
From: Major Hayden <major@redhat.com>
Date: Fri, 5 Jun 2020 09:30:48 -0500
Subject: [PATCH] Use S3 for CI repositories

Upload artifacts from the mock builds into S3 to allow us to burst into
AWS when needed.

Signed-off-by: Major Hayden <major@redhat.com>
---
 schutzbot/Jenkinsfile  | 40 +++++++++++++++++++---------------------
 schutzbot/mockbuild.sh | 38 +++++++++++++++++++++++++++-----------
 2 files changed, 46 insertions(+), 32 deletions(-)

diff --git a/schutzbot/Jenkinsfile b/schutzbot/Jenkinsfile
index d85732ad..20b9acdb 100644
--- a/schutzbot/Jenkinsfile
+++ b/schutzbot/Jenkinsfile
@@ -2,11 +2,7 @@ pipeline {
     agent none
 
     environment {
-        // We upload build RPMs and repository files here.
-        OPENSHIFT_CONTAINER = "osbuildci-artifacts"
-        // The files we upload are available under this URL.
-        // This is auto-generated from OpenStack.
-        MOCK_REPO_BASE_URL = "https://rhos-d.infra.prod.upshift.rdu2.redhat.com:13808/v1/AUTH_95e858620fb34bcc9162d9f52367a560/osbuildci-artifacts"
+        AWS_REGION = "us-east-2"
     }
 
     options {
@@ -24,7 +20,7 @@ pipeline {
                 stage('Fedora 31') {
                     agent { label "fedora31" }
                     environment {
-                        OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
+                        AWS_CREDS = credentials('aws-credentials-osbuildci')
                     }
                     steps {
                         sh "schutzbot/mockbuild.sh"
@@ -37,7 +33,7 @@ pipeline {
                 stage('Fedora 32') {
                     agent { label "fedora32" }
                     environment {
-                        OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
+                        AWS_CREDS = credentials('aws-credentials-osbuildci')
                     }
                     steps {
                         sh "schutzbot/mockbuild.sh"
@@ -50,7 +46,7 @@ pipeline {
                 stage('RHEL 8 CDN') {
                     agent { label "rhel8" }
                     environment {
-                        OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
+                        AWS_CREDS = credentials('aws-credentials-osbuildci')
                     }
                     steps {
                         sh "schutzbot/mockbuild.sh"
@@ -60,19 +56,21 @@ pipeline {
                         )
                     }
                 }
-                stage('RHEL 8.3 Nightly') {
-                    agent { label "rhel83" }
-                    environment {
-                        OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
-                    }
-                    steps {
-                        sh "schutzbot/mockbuild.sh"
-                        stash (
-                            includes: 'osbuild-mock.repo',
-                            name: 'rhel83'
-                        )
-                    }
-                }
+                // NOTE(mhayden): Disabling this for now since we don't have
+                // access to these repositories in AWS.
+                // stage('RHEL 8.3 Nightly') {
+                //     agent { label "rhel83" }
+                //     environment {
+                //         OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
+                //     }
+                //     steps {
+                //         sh "schutzbot/mockbuild.sh"
+                //         stash (
+                //             includes: 'osbuild-mock.repo',
+                //             name: 'rhel83'
+                //         )
+                //     }
+                // }
             }
         }
         stage("Functional Testing") {
diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh
index 65875a58..c63d700e 100755
--- a/schutzbot/mockbuild.sh
+++ b/schutzbot/mockbuild.sh
@@ -9,12 +9,34 @@ function greenprint {
 # Get OS details.
 source /etc/os-release
 
-# Set variables.
-CONTAINER=osbuildci-artifacts
+# Install s3cmd if it is not present.
+if ! s3cmd --version; then
+    greenprint "📦 Installing s3cmd"
+    sudo pip3 install s3cmd
+fi
+
+# Jenkins sets a workspace variable as the root of its working directory.
 WORKSPACE=${WORKSPACE:-$(pwd)}
+
+# Mock configuration file to use for building RPMs.
 MOCK_CONFIG="${ID}-${VERSION_ID%.*}-$(uname -m)"
+
+# Jenkins takes the proposed PR and merges it onto master. Although this
+# creates a new SHA (which is slightly confusing), it ensures that the code
+# merges properly against master and it tests the code against the latest
+# commit in master, which is certainly good.
 POST_MERGE_SHA=$(git rev-parse --short HEAD)
+
+# Bucket in S3 where our artifacts are uploaded
+REPO_BUCKET=osbuild-composer-repos
+
+# Public URL for the S3 bucket with our artifacts.
+MOCK_REPO_BASE_URL="http://osbuild-composer-repos.s3-website.us-east-2.amazonaws.com"
+
+# Directory to hold the RPMs temporarily before we upload them.
 REPO_DIR=repo/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}
+
+# Full URL to the RPM repository after they are uploaded.
 REPO_URL=${MOCK_REPO_BASE_URL}/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}
 
 # Print some data.
@@ -56,16 +78,10 @@ mv ${REPO_DIR}/*.log $WORKSPACE
 greenprint "⛓️ Creating dnf repository"
 createrepo_c ${REPO_DIR}
 
-# Prepare to upload to swift.
-greenprint "🛂 Setting up OpenStack authentication credentials"
-mkdir -p ~/.config/openstack
-cp $OPENSTACK_CREDS ~/.config/openstack/clouds.yml
-export OS_CLOUD=psi
-
-# Upload repository to swift.
-greenprint "☁ Uploading RPMs to OpenStack object storage"
+# Upload repository to S3.
+greenprint "☁ Uploading RPMs to S3"
 pushd repo
-    find * -type f -print | xargs openstack object create -f value $CONTAINER
+     s3cmd --acl-public sync . s3://${REPO_BUCKET}/
 popd
 
 # Create a repository file.