pipeline { agent none environment { AWS_REGION = "us-east-2" AWS_BUCKET = "imagebuilder-jenkins-testing-use2" // Colorful Ansible always looks nicer. ANSIBLE_FORCE_COLOR="True" // Time each task and display stdout as YAML (easier to read). ANSIBLE_LOAD_CALLBACK_PLUGINS="True" ANSIBLE_CALLBACK_WHITELIST="profile_tasks" ANSIBLE_STDOUT_CALLBACK="yaml" // Don't display those ugly purple deprecation warnings. ANSIBLE_DEPRECATION_WARNINGS="False" // Our host keys are constantly changing. ANSIBLE_HOST_KEY_CHECKING="False" // Enable ssh pipelining for faster deployments to remote nodes. ANSIBLE_PIPELINING="True" } options { timestamps() ansiColor('xterm') // Cancel the pipeline if it runs for more than three hours. timeout( time: 3, unit: "HOURS" ) } stages { stage("Mock") { // Halt the entire pipeline if a single RPM build fails. That // could indicate a code problem that needs to be investigated. failFast true parallel { stage('Fedora 31') { agent { label "fedora31" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { sh "schutzbot/ci_details.sh" sh "schutzbot/mockbuild.sh" stash ( includes: 'osbuild-mock.repo', name: 'fedora31' ) } } stage('Fedora 32') { agent { label "fedora32" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { sh "schutzbot/ci_details.sh" sh "schutzbot/mockbuild.sh" stash ( includes: 'osbuild-mock.repo', name: 'fedora32' ) } } stage('RHEL 8 CDN') { agent { label "rhel8" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { sh "schutzbot/ci_details.sh" sh "schutzbot/mockbuild.sh" stash ( includes: 'osbuild-mock.repo', name: 'rhel8cdn' ) } } // NOTE(mhayden): RHEL 8.3 is only available in PSI for now. stage('RHEL 8.3 Nightly') { agent { label "rhel83 && psi" } environment { AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { sh "schutzbot/mockbuild.sh" stash ( includes: 'osbuild-mock.repo', name: 'rhel83' ) } } } } stage("Functional Testing") { // Allow the other stages to finish if a single stage fails. failFast false parallel { stage('Fedora 31 base') { agent { label "fedora31" } environment { TEST_TYPE = "base" } steps { unstash 'fedora31' run_tests('base') } post { always { preserve_logs('fedora31-base') } } } stage('Fedora 31 image') { agent { label "fedora31 && psi" } environment { TEST_TYPE = "image" AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { unstash 'fedora31' run_tests('image') } post { always { preserve_logs('fedora31-image') } } } stage('Fedora 31 integration') { agent { label "fedora31 && psi" } environment { TEST_TYPE = "integration" AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { unstash 'fedora31' run_tests('integration') } post { always { preserve_logs('fedora31-integration') } } } stage('Fedora 32 base') { agent { label "fedora32" } environment { TEST_TYPE = "base" } steps { unstash 'fedora32' run_tests('base') } post { always { preserve_logs('fedora32-base') } } } stage('Fedora 32 image') { agent { label "fedora32 && psi" } environment { TEST_TYPE = "image" AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { unstash 'fedora32' run_tests('image') } post { always { preserve_logs('fedora32-image') } } } stage('Fedora 32 integration') { agent { label "fedora32" } environment { TEST_TYPE = "integration" AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { unstash 'fedora32' run_tests('integration') } post { always { preserve_logs('fedora32-integration') } } } stage('RHEL 8 CDN Base') { agent { label "rhel8" } environment { TEST_TYPE = "base" } steps { unstash 'rhel8cdn' run_tests('base') } post { always { preserve_logs('rhel8-base') } } } stage('RHEL 8 CDN Image') { agent { label "rhel8 && psi" } environment { TEST_TYPE = "image" AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { unstash 'rhel8cdn' run_tests('image') } post { always { preserve_logs('rhel8-image') } } } stage('RHEL 8 CDN integration') { agent { label "rhel8" } environment { TEST_TYPE = "integration" AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { unstash 'rhel8cdn' run_tests('integration') } post { always { preserve_logs('rhel8-integration') } } } stage('RHEL 8.3 Base') { agent { label "rhel83" } environment { TEST_TYPE = "base" } steps { unstash 'rhel83' // If RHEL 8.3 fails, let the stage fail without // marking the entire pipeline as a failure. Once // RHEL 8.3 calms down and tests begin to pass, // remove this wrapper and make this test look like // the others. catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { run_tests('base') } } post { always { preserve_logs('rhel83-base') } } } stage('RHEL 8.3 Image') { agent { label "rhel83 && psi" } environment { TEST_TYPE = "image" AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { unstash 'rhel83' // If RHEL 8.3 fails, let the stage fail without // marking the entire pipeline as a failure. Once // RHEL 8.3 calms down and tests begin to pass, // remove this wrapper and make this test look like // the others. catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { run_tests('image') } } post { always { preserve_logs('rhel83-image') } } } stage('RHEL 8.3 integration') { agent { label "rhel83" } environment { TEST_TYPE = "integration" AWS_CREDS = credentials('aws-credentials-osbuildci') } steps { unstash 'rhel83' // If RHEL 8.3 fails, let the stage fail without // marking the entire pipeline as a failure. Once // RHEL 8.3 calms down and tests begin to pass, // remove this wrapper and make this test look like // the others. catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { run_tests('integration') } } post { always { preserve_logs('rhel83-integration') } } } } } } } // Set up a function to hold the steps needed to run the tests so we don't // need to copy/paste the same lines over and over above. void run_tests(test_type) { // Get CI machine details. sh ( label: "Get CI machine details", script: "schutzbot/ci_details.sh" ) // Run the tests from the repository. sh ( label: "Deploy", script: "schutzbot/run_tests.sh" ) // Run the base tests. if (test_type == 'base') { sh ( label: "Base tests", script: "schutzbot/run_base_tests.sh" ) } if (test_type == 'image') { sh ( label: "Image tests", script: "schutzbot/run_image_tests.sh" ) } if (test_type == 'integration') { // Run the qcow2 test. sh ( label: "Integration test: QCOW2", script: "test/image-tests/qemu.sh qcow2" ) // Run the openstack test. sh ( label: "Integration test: OpenStack", script: "test/image-tests/qemu.sh openstack" ) // Run the VHD/Azure test. sh ( label: "Integration test: VHD", script: "test/image-tests/qemu.sh vhd" ) // Run the VMDK/VMware test. sh ( label: "Integration test: VMDK", script: "test/image-tests/qemu.sh vmdk" ) // Run the AWS test. sh ( label: "Integration test: AWS", script: "test/image-tests/aws.sh" ) } } // Move logs to a unique location and tell Jenkins to capture them on success // or failure. void preserve_logs(test_slug) { // Save the systemd journal. sh "journalctl --boot > systemd-journald.log" // Make a directory for the log files and move the logs there. sh "mkdir ${test_slug} && mv *.log *.jpg ${test_slug}/ || true" // Artifact the logs. archiveArtifacts ( allowEmptyArchive: true, artifacts: "${test_slug}/*.log,${test_slug}/*.jpg" ) }