particle-os/debian-forge
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Use S3 for CI repositories

Browse source 
Upload artifacts from the mock builds into S3 to allow us to burst into
AWS when needed.

Signed-off-by: Major Hayden <major@redhat.com>
This commit is contained in:
Major Hayden 2020-06-05 09:30:48 -05:00 • committed by Major Hayden
parent 5c3ce5c30a
commit 381abb354a
2 changed files with 46 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

40
schutzbot/Jenkinsfile vendored
View file

@ -2,11 +2,7 @@ pipeline {
agent none
environment {
// We upload build RPMs and repository files here.
OPENSHIFT_CONTAINER = "osbuildci-artifacts"
// The files we upload are available under this URL.
// This is auto-generated from OpenStack.
MOCK_REPO_BASE_URL = "https://rhos-d.infra.prod.upshift.rdu2.redhat.com:13808/v1/AUTH_95e858620fb34bcc9162d9f52367a560/osbuildci-artifacts"
AWS_REGION = "us-east-2"
}
options {
@ -24,7 +20,7 @@ pipeline {
stage('Fedora 31') {
agent { label "fedora31" }
environment {
OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/mockbuild.sh"
@ -37,7 +33,7 @@ pipeline {
stage('Fedora 32') {
agent { label "fedora32" }
environment {
OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/mockbuild.sh"
@ -50,7 +46,7 @@ pipeline {
stage('RHEL 8 CDN') {
agent { label "rhel8" }
environment {
OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/mockbuild.sh"
@ -60,19 +56,21 @@ pipeline {
)
}
}
stage('RHEL 8.3 Nightly') {
agent { label "rhel83" }
environment {
OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
}
steps {
sh "schutzbot/mockbuild.sh"
stash (
includes: 'osbuild-mock.repo',
name: 'rhel83'
)
}
}
// NOTE(mhayden): Disabling this for now since we don't have
// access to these repositories in AWS.
// stage('RHEL 8.3 Nightly') {
// agent { label "rhel83" }
// environment {
// OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
// }
// steps {
// sh "schutzbot/mockbuild.sh"
// stash (
// includes: 'osbuild-mock.repo',
// name: 'rhel83'
// )
// }
// }
}
}
stage("Functional Testing") {

38
schutzbot/mockbuild.sh
View file

@ -9,12 +9,34 @@ function greenprint {
# Get OS details.
source /etc/os-release
# Set variables.
CONTAINER=osbuildci-artifacts
# Install s3cmd if it is not present.
if ! s3cmd --version; then
greenprint "📦 Installing s3cmd"
sudo pip3 install s3cmd
fi
# Jenkins sets a workspace variable as the root of its working directory.
WORKSPACE=${WORKSPACE:-$(pwd)}
# Mock configuration file to use for building RPMs.
MOCK_CONFIG="${ID}-${VERSION_ID%.*}-$(uname -m)"
# Jenkins takes the proposed PR and merges it onto master. Although this
# creates a new SHA (which is slightly confusing), it ensures that the code
# merges properly against master and it tests the code against the latest
# commit in master, which is certainly good.
POST_MERGE_SHA=$(git rev-parse --short HEAD)
# Bucket in S3 where our artifacts are uploaded
REPO_BUCKET=osbuild-composer-repos
# Public URL for the S3 bucket with our artifacts.
MOCK_REPO_BASE_URL="http://osbuild-composer-repos.s3-website.us-east-2.amazonaws.com"
# Directory to hold the RPMs temporarily before we upload them.
REPO_DIR=repo/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}
# Full URL to the RPM repository after they are uploaded.
REPO_URL=${MOCK_REPO_BASE_URL}/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}
# Print some data.
@ -56,16 +78,10 @@ mv ${REPO_DIR}/*.log $WORKSPACE
greenprint "⛓️ Creating dnf repository"
createrepo_c ${REPO_DIR}
# Prepare to upload to swift.
greenprint "🛂 Setting up OpenStack authentication credentials"
mkdir -p ~/.config/openstack
cp $OPENSTACK_CREDS ~/.config/openstack/clouds.yml
export OS_CLOUD=psi
# Upload repository to swift.
greenprint "☁ Uploading RPMs to OpenStack object storage"
# Upload repository to S3.
greenprint "☁ Uploading RPMs to S3"
pushd repo
find * -type f -print | xargs openstack object create -f value $CONTAINER
s3cmd --acl-public sync . s3://${REPO_BUCKET}/
popd
# Create a repository file.