🏇🏻 Speed up pipeline with parallelism (again)

Now that we're going about multi-arch testing in a different way, we can
restore some of the old parallelism that we used to have in the pipeline
so it will finish faster.

Signed-off-by: Major Hayden <major@redhat.com>
This commit is contained in:
Major Hayden 2020-07-17 11:31:50 -05:00 committed by Major Hayden
parent 60aafdaaab
commit 20fd6521fb

673
schutzbot/Jenkinsfile vendored
View file

@ -16,6 +16,7 @@ pipeline {
)
}
stages {
stage("Prepare") {
agent { label "schutzbot" }
steps {
@ -26,452 +27,276 @@ pipeline {
}
}
// The outermost testing stage consists of multiple parallel lanes
// that allow tests to keep going with a single OS/arch combination
// without waiting on others. This is handy if one OS has slow mirrors
// or the arch is generally slower, like aarch64.
stage("Test 🤔") {
stage("Mock build ⚒") {
// Halt the pipeline immediately if a single mock build fails.
// A failure to build an RPM is serious and must be
// investigated.
failFast true
parallel {
// Fedora 31 on x86 has a full suite of tests and can run in
// AWS or PSI.
stage("F31 x86") {
stages {
stage('Mock build') {
agent { label "f31cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora31'
)
}
stage('F31') {
agent { label "f31cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stage('Base') {
agent { label "f31cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora31'
run_tests('base')
}
post {
always {
preserve_logs('fedora31-base')
}
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora31'
)
}
}
stage('F32') {
agent { label "f32cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stage('Image') {
agent { label "f31cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AZURE_CREDS = credentials('azure')
}
steps {
unstash 'fedora31'
run_tests('image')
}
post {
always {
preserve_logs('fedora31-image')
}
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora32'
)
}
}
stage('EL8') {
agent { label "rhel8cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stage('Integration') {
agent { label "f31cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora31'
run_tests('integration')
}
post {
always {
preserve_logs('fedora31-integration')
}
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel8cdn'
)
}
}
stage('EL8.3') {
agent { label "rhel83cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
NIGHTLY_REPO = credentials('rhel8-nightly-repo')
NIGHTLY_MOCK_TEMPLATE = credentials('rhel8-nightly-mock-template')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel83'
)
}
}
}
}
stage("Testing 🍌") {
parallel {
stage('F31 Base') {
agent { label "f31cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora31'
run_tests('base')
}
post {
always {
preserve_logs('fedora31-base')
}
}
}
stage('F31 Image') {
agent { label "f31cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AZURE_CREDS = credentials('azure')
}
steps {
unstash 'fedora31'
run_tests('image')
}
post {
always {
preserve_logs('fedora31-image')
}
}
}
stage('F31 Integration') {
agent { label "f31cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora31'
run_tests('integration')
}
post {
always {
preserve_logs('fedora31-integration')
}
}
}
// Fedora 31 on aarch64 has base tests enabled and can only
// run at AWS.
// stage("F31 aarch64") {
// stages {
// stage('Mock build') {
// agent { label "f31cloudbase_aarch64_temporary" }
// environment {
// AWS_CREDS = credentials('aws-credentials-osbuildci')
// }
// steps {
// sh "schutzbot/ci_details.sh"
// retry(3) {
// sh "schutzbot/mockbuild.sh"
// }
// stash (
// includes: 'osbuild-mock.repo',
// name: 'fedora31_aarch64'
// )
// }
// }
// stage('Base') {
// agent { label "f31cloudbase_aarch64_temporary" }
// environment { TEST_TYPE = "base" }
// steps {
// unstash 'fedora31_aarch64'
// run_tests('base')
// }
// post {
// always {
// preserve_logs('fedora31-base-aarch64')
// }
// }
// }
// stage('Image') {
// agent { label "f31cloudbase_aarch64_temporary" }
// environment {
// TEST_TYPE = "image"
// AWS_CREDS = credentials('aws-credentials-osbuildci')
// }
// steps {
// unstash 'fedora31_aarch64'
// run_tests('image')
// }
// post {
// always {
// preserve_logs('fedora31-image-aarch64')
// }
// }
// }
// }
// }
// Fedora 32 on x86 has a full suite of tests and can run in
// AWS or PSI.
stage("F32 x86") {
stages {
stage('Mock build') {
agent { label "f32cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora32'
)
}
stage('F32 Base') {
agent { label "f32cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora32'
run_tests('base')
}
post {
always {
preserve_logs('fedora32-base')
}
stage('Base') {
agent { label "f32cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora32'
run_tests('base')
}
post {
always {
preserve_logs('fedora32-base')
}
}
}
}
stage('F32 Image') {
agent { label "f32cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AZURE_CREDS = credentials('azure')
}
steps {
unstash 'fedora32'
run_tests('image')
}
post {
always {
preserve_logs('fedora32-image')
}
stage('Image') {
agent { label "f32cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AZURE_CREDS = credentials('azure')
}
steps {
unstash 'fedora32'
run_tests('image')
}
post {
always {
preserve_logs('fedora32-image')
}
}
}
stage('Integration') {
agent { label "f32cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora32'
run_tests('integration')
}
post {
always {
preserve_logs('fedora32-integration')
}
}
}
}
stage('F32 Integration') {
agent { label "f32cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora32'
run_tests('integration')
}
post {
always {
preserve_logs('fedora32-integration')
}
}
}
// Fedora 32 on aarch64 has base tests enabled and can only
// run at AWS.
// stage("F32 aarch64") {
// stages {
// stage('Mock build') {
// agent { label "f32cloudbase_aarch64_temporary" }
// environment {
// AWS_CREDS = credentials('aws-credentials-osbuildci')
// }
// steps {
// sh "schutzbot/ci_details.sh"
// retry(3) {
// sh "schutzbot/mockbuild.sh"
// }
// stash (
// includes: 'osbuild-mock.repo',
// name: 'fedora32_aarch64'
// )
// }
// }
// stage('Base') {
// agent { label "f32cloudbase_aarch64_temporary" }
// environment { TEST_TYPE = "base" }
// steps {
// unstash 'fedora32_aarch64'
// run_tests('base')
// }
// post {
// always {
// preserve_logs('fedora32-base-aarch64')
// }
// }
// }
// stage('Image') {
// agent { label "f32cloudbase_aarch64_temporary" }
// environment {
// TEST_TYPE = "image"
// AWS_CREDS = credentials('aws-credentials-osbuildci')
// }
// steps {
// unstash 'fedora32_aarch64'
// run_tests('image')
// }
// post {
// always {
// preserve_logs('fedora32-image-aarch64')
// }
// }
// }
// }
// }
// RHEL 8 on x86 has a full suite of tests and can run in
// AWS or PSI.
stage("RHEL8 x86") {
stages {
stage('Mock build') {
agent { label "rhel8cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel8cdn'
)
}
stage('EL8 Base') {
agent { label "rhel8cloudbase && x86_64" }
environment {
TEST_TYPE = "base"
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('base')
}
post {
always {
preserve_logs('rhel8-base')
}
stage('Base') {
agent { label "rhel8cloudbase && x86_64" }
environment {
TEST_TYPE = "base"
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('base')
}
post {
always {
preserve_logs('rhel8-base')
}
}
}
}
stage('EL8 Image') {
agent { label "rhel8cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AZURE_CREDS = credentials('azure')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('image')
}
post {
always {
preserve_logs('rhel8-image')
}
stage('Image') {
agent { label "rhel8cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AZURE_CREDS = credentials('azure')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('image')
}
post {
always {
preserve_logs('rhel8-image')
}
}
}
stage('Integration') {
agent { label "rhel8cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('integration')
}
post {
always {
preserve_logs('rhel8-integration')
}
}
}
}
stage('EL8 Integration') {
agent { label "rhel8cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('integration')
}
post {
always {
preserve_logs('rhel8-integration')
}
}
}
// RHEL 8 on aarch64 has base tests enabled and can only
// run at AWS.
// stage("RHEL8 aarch64") {
// stages {
// stage('Mock build') {
// agent { label "rhel8cloudbase_aarch64_temporary" }
// environment {
// AWS_CREDS = credentials('aws-credentials-osbuildci')
// RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production-aarch64')
// }
// steps {
// sh "schutzbot/ci_details.sh"
// retry(3) {
// sh "schutzbot/mockbuild.sh"
// }
// stash (
// includes: 'osbuild-mock.repo',
// name: 'rhel8cdn_aarch64'
// )
// }
// }
// stage('Base') {
// agent { label "rhel8cloudbase_aarch64_temporary" }
// environment {
// TEST_TYPE = "base"
// RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production-aarch64')
// }
// steps {
// unstash 'rhel8cdn_aarch64'
// run_tests('base')
// }
// post {
// always {
// preserve_logs('rhel8-base-aarch64')
// }
// }
// }
// stage('Image') {
// agent { label "rhel8cloudbase_aarch64_temporary" }
// environment {
// TEST_TYPE = "image"
// RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production-aarch64')
// }
// steps {
// unstash 'rhel8cdn_aarch64'
// run_tests('image')
// }
// post {
// always {
// preserve_logs('rhel8-image-aarch64')
// }
// }
// }
// }
// }
// RHEL 8.3 on x86 has a full suite of tests and can only run
// in PSI until 8.3 beta content appears on the public CDN.
stage("RHEL8.3 x86") {
stages {
stage('Mock build') {
agent { label "rhel83cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
NIGHTLY_REPO = credentials('rhel8-nightly-repo')
NIGHTLY_MOCK_TEMPLATE = credentials('rhel8-nightly-mock-template')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel83'
)
}
stage('EL8.3 Base') {
agent { label "rhel83cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'rhel83'
run_tests('base')
}
post {
always {
preserve_logs('rhel83-base')
}
stage('Base') {
agent { label "rhel83cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'rhel83'
run_tests('base')
}
post {
always {
preserve_logs('rhel83-base')
}
}
}
}
stage('EL8.3 Image') {
agent { label "rhel83cloudbase && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AZURE_CREDS = credentials('azure')
}
steps {
unstash 'rhel83'
run_tests('image')
}
post {
always {
preserve_logs('rhel83-image')
}
stage('Image') {
agent { label "rhel83cloudbase && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AZURE_CREDS = credentials('azure')
}
steps {
unstash 'rhel83'
run_tests('image')
}
post {
always {
preserve_logs('rhel83-image')
}
}
}
stage('Integration') {
agent { label "rhel83cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'rhel83'
run_tests('integration')
}
post {
always {
preserve_logs('rhel83-integration')
}
}
}
}
stage('EL8.3 Integration') {
agent { label "rhel83cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'rhel83'
run_tests('integration')
}
post {
always {
preserve_logs('rhel83-integration')
}
}
}