🦾 Add aarch64 base tests; restructure stages

Add base aarch64 tests for Fedora 31/32 and RHEL 8 CDN.

Also restructure the pipeline so that a slow distro or slow architecture
won't hold up the faster-moving tests.

This helps a bit more with #834.

Signed-off-by: Major Hayden <major@redhat.com>
This commit is contained in:
Major Hayden 2020-07-13 16:10:43 -05:00 committed by Tom Gundersen
parent bc1bca1538
commit 45c159e7f8

644
schutzbot/Jenkinsfile vendored
View file

@ -4,18 +4,6 @@ pipeline {
environment {
AWS_REGION = "us-east-2"
AWS_BUCKET = "imagebuilder-jenkins-testing-use2"
// Colorful Ansible always looks nicer.
ANSIBLE_FORCE_COLOR="True"
// Time each task and display stdout as YAML (easier to read).
ANSIBLE_LOAD_CALLBACK_PLUGINS="True"
ANSIBLE_CALLBACK_WHITELIST="profile_tasks"
ANSIBLE_STDOUT_CALLBACK="yaml"
// Don't display those ugly purple deprecation warnings.
ANSIBLE_DEPRECATION_WARNINGS="False"
// Our host keys are constantly changing.
ANSIBLE_HOST_KEY_CHECKING="False"
// Enable ssh pipelining for faster deployments to remote nodes.
ANSIBLE_PIPELINING="True"
}
options {
@ -38,318 +26,370 @@ pipeline {
}
}
stage("Mock") {
// Halt the entire pipeline if a single RPM build fails. That
// could indicate a code problem that needs to be investigated.
failFast true
// The outermost testing stage consists of multiple parallel lanes
// that allow tests to keep going with a single OS/arch combination
// without waiting on others. This is handy if one OS has slow mirrors
// or the arch is generally slower, like aarch64.
stage("Test 🤔") {
parallel {
stage('Fedora 31') {
agent { label "f31cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
// Fedora 31 on x86 has a full suite of tests and can run in
// AWS or PSI.
stage("F31 x86") {
stages {
stage('Mock build') {
agent { label "f31cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora31'
)
}
}
stage('Base') {
agent { label "f31cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora31'
run_tests('base')
}
post {
always {
preserve_logs('fedora31-base')
}
}
}
stage('Image') {
agent { label "f31cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora31'
run_tests('image')
}
post {
always {
preserve_logs('fedora31-image')
}
}
}
stage('Integration') {
agent { label "f31cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora31'
run_tests('integration')
}
post {
always {
preserve_logs('fedora31-integration')
}
}
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora31'
)
}
}
stage('Fedora 31 aarch64') {
agent { label "f31cloudbase_aarch64_temporary" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora31_aarch64'
)
}
}
stage('Fedora 32') {
agent { label "f32cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora32'
)
}
}
stage('Fedora 32 aarch64') {
agent { label "f32cloudbase_aarch64_temporary" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora32_aarch64'
)
}
}
stage('RHEL 8 CDN') {
agent { label "rhel8cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel8cdn'
)
}
}
stage('RHEL 8 CDN aarch64') {
agent { label "rhel8cloudbase_aarch64_temporary" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production-aarch64')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel8cdn_aarch64'
)
}
}
// NOTE(mhayden): RHEL 8.3 is only available in PSI for now.
stage('RHEL 8.3 Nightly') {
agent { label "rhel83cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
NIGHTLY_REPO = credentials('rhel8-nightly-repo')
NIGHTLY_MOCK_TEMPLATE = credentials('rhel8-nightly-mock-template')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel83'
)
}
}
}
}
stage("Functional Testing") {
// Allow the other stages to finish if a single stage fails.
failFast false
parallel {
stage('Fedora 31 base') {
agent { label "f31cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora31'
run_tests('base')
}
post {
always {
preserve_logs('fedora31-base')
// Fedora 31 on aarch64 has base tests enabled and can only
// run at AWS.
stage("F31 aarch64") {
stages {
stage('Mock build') {
agent { label "f31cloudbase_aarch64_temporary" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora31_aarch64'
)
}
}
stage('Base') {
agent { label "f31cloudbase_aarch64_temporary" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora31_aarch64'
run_tests('base')
}
post {
always {
preserve_logs('fedora31-base')
}
}
}
}
}
stage('Fedora 31 image') {
agent { label "f31cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora31'
run_tests('image')
}
post {
always {
preserve_logs('fedora31-image')
// Fedora 32 on x86 has a full suite of tests and can run in
// AWS or PSI.
stage("F32 x86") {
stages {
stage('Mock build') {
agent { label "f32cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora32'
)
}
}
stage('Base') {
agent { label "f32cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora32'
run_tests('base')
}
post {
always {
preserve_logs('fedora32-base')
}
}
}
stage('Image') {
agent { label "f32cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora32'
run_tests('image')
}
post {
always {
preserve_logs('fedora32-image')
}
}
}
stage('Integration') {
agent { label "f32cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora32'
run_tests('integration')
}
post {
always {
preserve_logs('fedora32-integration')
}
}
}
}
}
stage('Fedora 31 integration') {
agent { label "f31cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora31'
run_tests('integration')
}
post {
always {
preserve_logs('fedora31-integration')
// Fedora 32 on aarch64 has base tests enabled and can only
// run at AWS.
stage("F32 aarch64") {
stages {
stage('Mock build') {
agent { label "f32cloudbase_aarch64_temporary" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'fedora32_aarch64'
)
}
}
stage('Base') {
agent { label "f32cloudbase_aarch64_temporary" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora32_aarch64'
run_tests('base')
}
post {
always {
preserve_logs('fedora32-base')
}
}
}
}
}
stage('Fedora 32 base') {
agent { label "f32cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'fedora32'
run_tests('base')
}
post {
always {
preserve_logs('fedora32-base')
// RHEL 8 on x86 has a full suite of tests and can run in
// AWS or PSI.
stage("RHEL8 x86") {
stages {
stage('Mock build') {
agent { label "rhel8cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel8cdn'
)
}
}
stage('Base') {
agent { label "rhel8cloudbase && x86_64" }
environment {
TEST_TYPE = "base"
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('base')
}
post {
always {
preserve_logs('rhel8-base')
}
}
}
stage('Image') {
agent { label "rhel8cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('image')
}
post {
always {
preserve_logs('rhel8-image')
}
}
}
stage('Integration') {
agent { label "rhel8cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('integration')
}
post {
always {
preserve_logs('rhel8-integration')
}
}
}
}
}
stage('Fedora 32 image') {
agent { label "f32cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora32'
run_tests('image')
}
post {
always {
preserve_logs('fedora32-image')
// RHEL 8 on aarch64 has base tests enabled and can only
// run at AWS.
stage("RHEL8 aarch64") {
stages {
stage('Mock build') {
agent { label "rhel8cloudbase_aarch64_temporary" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production-aarch64')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel8cdn_aarch64'
)
}
}
stage('Base') {
agent { label "rhel8cloudbase_aarch64_temporary" }
environment {
TEST_TYPE = "base"
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production-aarch64')
}
steps {
unstash 'rhel8cdn_aarch64'
run_tests('base')
}
post {
always {
preserve_logs('fedora32-base')
}
}
}
}
}
stage('Fedora 32 integration') {
agent { label "f32cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'fedora32'
run_tests('integration')
}
post {
always {
preserve_logs('fedora32-integration')
// RHEL 8.3 on x86 has a full suite of tests and can only run
// in PSI until 8.3 beta content appears on the public CDN.
stage("RHEL8.3 x86") {
stages {
stage('RHEL 8.3 Nightly') {
agent { label "rhel83cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
NIGHTLY_REPO = credentials('rhel8-nightly-repo')
NIGHTLY_MOCK_TEMPLATE = credentials('rhel8-nightly-mock-template')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel83'
)
}
}
}
}
stage('RHEL 8 CDN Base') {
agent { label "rhel8cloudbase && x86_64" }
environment {
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
TEST_TYPE = "base"
}
steps {
unstash 'rhel8cdn'
run_tests('base')
}
post {
always {
preserve_logs('rhel8-base')
}
}
}
stage('RHEL 8 CDN Image') {
agent { label "rhel8cloudbase && psi && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'rhel8cdn'
run_tests('image')
}
post {
always {
preserve_logs('rhel8-image')
}
}
}
stage('RHEL 8 CDN integration') {
agent { label "rhel8cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests('integration')
}
post {
always {
preserve_logs('rhel8-integration')
}
}
}
stage('RHEL 8.3 Base') {
agent { label "rhel83cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'rhel83'
run_tests('base')
}
post {
always {
preserve_logs('rhel83-base')
}
}
}
stage('RHEL 8.3 Image') {
agent { label "rhel83cloudbase && x86_64" }
environment {
TEST_TYPE = "image"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'rhel83'
run_tests('image')
}
post {
always {
preserve_logs('rhel83-image')
}
}
}
stage('RHEL 8.3 integration') {
agent { label "rhel83cloudbase && x86_64" }
environment {
TEST_TYPE = "integration"
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
unstash 'rhel83'
run_tests('integration')
}
post {
always {
preserve_logs('rhel83-integration')
stage('RHEL 8.3 Base') {
agent { label "rhel83cloudbase && x86_64" }
environment { TEST_TYPE = "base" }
steps {
unstash 'rhel83'
run_tests('base')
}
post {
always {
preserve_logs('rhel83-base')
}
}
}
}
}
}