🦾 Add aarch64 base tests; restructure stages

Add base aarch64 tests for Fedora 31/32 and RHEL 8 CDN.

Also restructure the pipeline so that a slow distro or slow architecture
won't hold up the faster-moving tests.

This helps a bit more with #834.

Signed-off-by: Major Hayden <major@redhat.com>
This commit is contained in:
Major Hayden 2020-07-13 16:10:43 -05:00 committed by Tom Gundersen
parent bc1bca1538
commit 45c159e7f8

644
schutzbot/Jenkinsfile vendored
View file

@ -4,18 +4,6 @@ pipeline {
environment { environment {
AWS_REGION = "us-east-2" AWS_REGION = "us-east-2"
AWS_BUCKET = "imagebuilder-jenkins-testing-use2" AWS_BUCKET = "imagebuilder-jenkins-testing-use2"
// Colorful Ansible always looks nicer.
ANSIBLE_FORCE_COLOR="True"
// Time each task and display stdout as YAML (easier to read).
ANSIBLE_LOAD_CALLBACK_PLUGINS="True"
ANSIBLE_CALLBACK_WHITELIST="profile_tasks"
ANSIBLE_STDOUT_CALLBACK="yaml"
// Don't display those ugly purple deprecation warnings.
ANSIBLE_DEPRECATION_WARNINGS="False"
// Our host keys are constantly changing.
ANSIBLE_HOST_KEY_CHECKING="False"
// Enable ssh pipelining for faster deployments to remote nodes.
ANSIBLE_PIPELINING="True"
} }
options { options {
@ -38,318 +26,370 @@ pipeline {
} }
} }
stage("Mock") { // The outermost testing stage consists of multiple parallel lanes
// Halt the entire pipeline if a single RPM build fails. That // that allow tests to keep going with a single OS/arch combination
// could indicate a code problem that needs to be investigated. // without waiting on others. This is handy if one OS has slow mirrors
failFast true // or the arch is generally slower, like aarch64.
stage("Test 🤔") {
parallel { parallel {
stage('Fedora 31') { // Fedora 31 on x86 has a full suite of tests and can run in
agent { label "f31cloudbase && x86_64" } // AWS or PSI.
environment { stage("F31 x86") {
AWS_CREDS = credentials('aws-credentials-osbuildci') stages {
} stage('Mock build') {
steps { agent { label "f31cloudbase && x86_64" }
sh "schutzbot/ci_details.sh" environment {
retry(3) { AWS_CREDS = credentials('aws-credentials-osbuildci')
sh "schutzbot/mockbuild.sh" }
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora31'
)
}
}
stage('Base') {
agent { label "f31cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora31'
run_tests('base')
}
post {
always {
preserve_logs('fedora31-base')
}
}
}
stage('Image') {
agent { label "f31cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora31'
run_tests('image')
}
post {
always {
preserve_logs('fedora31-image')
}
}
}
stage('Integration') {
agent { label "f31cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora31'
run_tests('integration')
}
post {
always {
preserve_logs('fedora31-integration')
}
}
} }
stash (
includes: 'osbuild-mock.repo',
name: 'fedora31'
)
} }
} }
stage('Fedora 31 aarch64') {
agent { label "f31cloudbase_aarch64_temporary" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora31_aarch64'
)
}
}
stage('Fedora 32') {
agent { label "f32cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora32'
)
}
}
stage('Fedora 32 aarch64') {
agent { label "f32cloudbase_aarch64_temporary" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora32_aarch64'
)
}
}
stage('RHEL 8 CDN') {
agent { label "rhel8cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel8cdn'
)
}
}
stage('RHEL 8 CDN aarch64') {
agent { label "rhel8cloudbase_aarch64_temporary" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production-aarch64')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel8cdn_aarch64'
)
}
}
// NOTE(mhayden): RHEL 8.3 is only available in PSI for now.
stage('RHEL 8.3 Nightly') {
agent { label "rhel83cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
NIGHTLY_REPO = credentials('rhel8-nightly-repo')
NIGHTLY_MOCK_TEMPLATE = credentials('rhel8-nightly-mock-template')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel83'
)
}
}
}
}
stage("Functional Testing") {
// Allow the other stages to finish if a single stage fails.
failFast false
parallel { // Fedora 31 on aarch64 has base tests enabled and can only
stage('Fedora 31 base') { // run at AWS.
agent { label "f31cloudbase && x86_64" } stage("F31 aarch64") {
environment { TEST_TYPE = "base" } stages {
steps { stage('Mock build') {
unstash 'fedora31' agent { label "f31cloudbase_aarch64_temporary" }
run_tests('base') environment {
} AWS_CREDS = credentials('aws-credentials-osbuildci')
post { }
always { steps {
preserve_logs('fedora31-base') sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora31_aarch64'
)
}
}
stage('Base') {
agent { label "f31cloudbase_aarch64_temporary" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora31_aarch64'
run_tests('base')
}
post {
always {
preserve_logs('fedora31-base')
}
}
} }
} }
} }
stage('Fedora 31 image') {
agent { label "f31cloudbase && psi && x86_64" } // Fedora 32 on x86 has a full suite of tests and can run in
environment { // AWS or PSI.
TEST_TYPE = "image" stage("F32 x86") {
AWS_CREDS = credentials('aws-credentials-osbuildci') stages {
} stage('Mock build') {
steps { agent { label "f32cloudbase && x86_64" }
unstash 'fedora31' environment {
run_tests('image') AWS_CREDS = credentials('aws-credentials-osbuildci')
} }
post { steps {
always { sh "schutzbot/ci_details.sh"
preserve_logs('fedora31-image') retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora32'
)
}
}
stage('Base') {
agent { label "f32cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora32'
run_tests('base')
}
post {
always {
preserve_logs('fedora32-base')
}
}
}
stage('Image') {
agent { label "f32cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora32'
run_tests('image')
}
post {
always {
preserve_logs('fedora32-image')
}
}
}
stage('Integration') {
agent { label "f32cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora32'
run_tests('integration')
}
post {
always {
preserve_logs('fedora32-integration')
}
}
} }
} }
} }
stage('Fedora 31 integration') {
agent { label "f31cloudbase && x86_64" } // Fedora 32 on aarch64 has base tests enabled and can only
environment { // run at AWS.
TEST_TYPE = "integration" stage("F32 aarch64") {
AWS_CREDS = credentials('aws-credentials-osbuildci') stages {
} stage('Mock build') {
steps { agent { label "f32cloudbase_aarch64_temporary" }
unstash 'fedora31' environment {
run_tests('integration') AWS_CREDS = credentials('aws-credentials-osbuildci')
} }
post { steps {
always { sh "schutzbot/ci_details.sh"
preserve_logs('fedora31-integration') retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora32_aarch64'
)
}
}
stage('Base') {
agent { label "f32cloudbase_aarch64_temporary" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora32_aarch64'
run_tests('base')
}
post {
always {
preserve_logs('fedora32-base')
}
}
} }
} }
} }
stage('Fedora 32 base') {
agent { label "f32cloudbase && x86_64" } // RHEL 8 on x86 has a full suite of tests and can run in
environment { TEST_TYPE = "base" } // AWS or PSI.
steps { stage("RHEL8 x86") {
unstash 'fedora32' stages {
run_tests('base') stage('Mock build') {
} agent { label "rhel8cloudbase && x86_64" }
post { environment {
always { AWS_CREDS = credentials('aws-credentials-osbuildci')
preserve_logs('fedora32-base') RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel8cdn'
)
}
}
stage('Base') {
agent { label "rhel8cloudbase && x86_64" }
environment {
TEST_TYPE = "base"
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('base')
}
post {
always {
preserve_logs('rhel8-base')
}
}
}
stage('Image') {
agent { label "rhel8cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('image')
}
post {
always {
preserve_logs('rhel8-image')
}
}
}
stage('Integration') {
agent { label "rhel8cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('integration')
}
post {
always {
preserve_logs('rhel8-integration')
}
}
} }
} }
} }
stage('Fedora 32 image') {
agent { label "f32cloudbase && psi && x86_64" } // RHEL 8 on aarch64 has base tests enabled and can only
environment { // run at AWS.
TEST_TYPE = "image" stage("RHEL8 aarch64") {
AWS_CREDS = credentials('aws-credentials-osbuildci') stages {
} stage('Mock build') {
steps { agent { label "rhel8cloudbase_aarch64_temporary" }
unstash 'fedora32' environment {
run_tests('image') AWS_CREDS = credentials('aws-credentials-osbuildci')
} RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production-aarch64')
post { }
always { steps {
preserve_logs('fedora32-image') sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel8cdn_aarch64'
)
}
}
stage('Base') {
agent { label "rhel8cloudbase_aarch64_temporary" }
environment {
TEST_TYPE = "base"
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production-aarch64')
}
steps {
unstash 'rhel8cdn_aarch64'
run_tests('base')
}
post {
always {
preserve_logs('fedora32-base')
}
}
} }
} }
} }
stage('Fedora 32 integration') {
agent { label "f32cloudbase && x86_64" } // RHEL 8.3 on x86 has a full suite of tests and can only run
environment { // in PSI until 8.3 beta content appears on the public CDN.
TEST_TYPE = "integration" stage("RHEL8.3 x86") {
AWS_CREDS = credentials('aws-credentials-osbuildci') stages {
} stage('RHEL 8.3 Nightly') {
steps { agent { label "rhel83cloudbase && x86_64" }
unstash 'fedora32' environment {
run_tests('integration') AWS_CREDS = credentials('aws-credentials-osbuildci')
} NIGHTLY_REPO = credentials('rhel8-nightly-repo')
post { NIGHTLY_MOCK_TEMPLATE = credentials('rhel8-nightly-mock-template')
always { }
preserve_logs('fedora32-integration') steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel83'
)
}
} }
} stage('RHEL 8.3 Base') {
} agent { label "rhel83cloudbase && x86_64" }
stage('RHEL 8 CDN Base') { environment { TEST_TYPE = "base" }
agent { label "rhel8cloudbase && x86_64" } steps {
environment { unstash 'rhel83'
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production') run_tests('base')
TEST_TYPE = "base" }
} post {
steps { always {
unstash 'rhel8cdn' preserve_logs('rhel83-base')
run_tests('base') }
} }
post {
always {
preserve_logs('rhel8-base')
}
}
}
stage('RHEL 8 CDN Image') {
agent { label "rhel8cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'rhel8cdn'
run_tests('image')
}
post {
always {
preserve_logs('rhel8-image')
}
}
}
stage('RHEL 8 CDN integration') {
agent { label "rhel8cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('integration')
}
post {
always {
preserve_logs('rhel8-integration')
}
}
}
stage('RHEL 8.3 Base') {
agent { label "rhel83cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'rhel83'
run_tests('base')
}
post {
always {
preserve_logs('rhel83-base')
}
}
}
stage('RHEL 8.3 Image') {
agent { label "rhel83cloudbase && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'rhel83'
run_tests('image')
}
post {
always {
preserve_logs('rhel83-image')
}
}
}
stage('RHEL 8.3 integration') {
agent { label "rhel83cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'rhel83'
run_tests('integration')
}
post {
always {
preserve_logs('rhel83-integration')
} }
} }
} }
} }