debian-forge-composer/schutzbot/Jenkinsfile
Martin Sehnoutka 4a54ab43e5 test/cases: Introduce common "regression" test for all distros
Create an entry point for all regression test called "regression.sh" and
run it as part of the base tests for all our distros. This entry
point contains logic for running only the test cases that are
appropriate for a given distribution.
2021-05-20 13:07:07 +03:00

912 lines
34 KiB
Groovy

def cron_string = BRANCH_NAME == "main" ? "@daily" : ""
pipeline {
agent none
// Don't run the nightly test on main. Main currently doesn't match the
// osbuild-composer version shipped in any RHEL versions.
// We should resume this when we start rebasing again.
// triggers {
// cron(cron_string)
// }
environment {
AWS_REGION = "us-east-2"
AWS_BUCKET = "imagebuilder-jenkins-testing-use2"
BUILD_CAUSE = detect_build_cause()
GCP_BUCKET = "osbuild-composer-testing"
GCP_REGION = "us-east4"
AZURE_TENANT_ID = "1710d22c-ccf0-4421-8ba7-0135cfaecb90"
AZURE_SUBSCRIPTION_ID = "8d026bb1-2a65-454d-a88f-c896db94c4f8"
AZURE_RESOURCE_GROUP = "sharing-research"
AZURE_LOCATION = "eastus"
}
options {
timestamps()
ansiColor('xterm')
// Cancel the pipeline if it runs for more than three hours.
timeout(
time: 12,
unit: "HOURS"
)
}
stages {
stage("Prepare 🤔") {
agent { label "schutzbot" }
options {
// Don't checkout the git repository here. It just clogs
// up the Jenkins disk space and does nothing for us.
skipDefaultCheckout()
}
steps {
sh (
label: "Get environment variables",
script: "env | sort"
)
}
}
stage("Mock build 👷🏻") {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
// Halt the pipeline immediately if a single mock build fails.
// A failure to build an RPM is serious and must be
// investigated.
failFast true
parallel {
stage('F32') {
agent { label "f32cloudbase && x86_64 && aws" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
}
}
stage('F33') {
agent { label "f33cloudbase && x86_64 && aws" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
}
}
stage('F33 aarch64') {
agent { label "f33cloudbase && aarch64 && aws" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
}
}
stage('F34') {
agent { label "f34cloudbase && x86_64 && aws" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
}
}
stage('F34 aarch64') {
agent { label "f34cloudbase && aarch64 && aws" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
}
}
stage('EL8') {
agent { label "rhel8cloudbase && x86_64 && aws" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
}
}
stage('EL8 aarch64') {
agent { label "rhel8cloudbase && aarch64 && aws" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
}
}
stage('EL8.4') {
agent { label "rhel84cloudbase && x86_64 && psi" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
RHEL84_NIGHTLY_REPO = credentials('rhel84-nightly-repo')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
}
}
stage('CS8') {
agent { label "cs8cloudbase && x86_64 && aws" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
}
}
stage('CS8 aarch64') {
agent { label "cs8cloudbase && aarch64 && aws" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
}
}
stage("Container build - x86_64") {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "f33cloudbase && x86_64 && aws" }
steps {
sh "schutzbot/containerbuild.sh"
}
}
}
}
stage("Prepare EL8 internal 🤔") {
agent { label "rhel8cloudbase && x86_64 && psi" }
when {
beforeAgent true
expression {
return env.BUILD_CAUSE == 'cron';
}
}
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
steps {
// don't build, use RPMs from the internal trees
sh "schutzbot/prepare-rhel-internal.sh"
stash (
includes: 'rhel8internal.repo',
name: 'rhel8internal'
)
stash (
includes: 'rhel-8.json',
name: 'rhel8json'
)
stash (
includes: 'rhel-8-beta.json',
name: 'rhel8betajson'
)
stash (
includes: 'COMPOSE_ID',
name: 'compose_id'
)
}
}
stage("Testing 🍌") {
parallel {
stage('F32 Base') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "f32cloudbase && x86_64 && aws" }
environment { TEST_TYPE = "base" }
steps {
run_tests('base')
}
post {
always {
preserve_logs('fedora32-base')
}
}
}
stage('F32 Integration') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "f32cloudbase && x86_64 && aws" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "fedora_32"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
AZURE_CLIENT_ID = credentials('azure-client-id')
AZURE_CLIENT_SECRET = credentials('azure-client-secret')
AZURE_CREDS = credentials('azure')
}
steps {
run_tests('integration')
}
post {
always {
preserve_logs('fedora32-integration')
sh (
label: "Run cloud cleaner just in case something failed",
script: "schutzbot/run_cloud_cleaner.sh"
)
}
}
}
stage('F32 OSTree') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "f32cloudbase && psi && x86_64" }
steps {
run_tests('ostree')
}
post {
always {
preserve_logs('fedora32-ostree')
}
}
}
stage('F33 Base') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "f33cloudbase && x86_64 && aws" }
environment { TEST_TYPE = "base" }
steps {
run_tests('base')
}
post {
always {
preserve_logs('fedora33-base')
}
}
}
stage('F33 Integration') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "f33cloudbase && x86_64 && aws" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "fedora_33"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
AZURE_CLIENT_ID = credentials('azure-client-id')
AZURE_CLIENT_SECRET = credentials('azure-client-secret')
AZURE_CREDS = credentials('azure')
}
steps {
run_tests('integration')
}
post {
always {
preserve_logs('fedora33-integration')
sh (
label: "Run cloud cleaner just in case something failed",
script: "schutzbot/run_cloud_cleaner.sh"
)
}
}
}
stage('F33 OSTree') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "f33cloudbase && psi && x86_64" }
steps {
run_tests('ostree')
}
post {
always {
preserve_logs('fedora33-ostree')
}
}
}
stage('F33 aarch64 Base') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "f33cloudbase && aarch64 && aws" }
environment { TEST_TYPE = "base" }
steps {
run_tests('base')
}
post {
always {
preserve_logs('fedora33-aarch64-base')
}
}
}
stage('F33: koji-osbuild') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "f33cloudbase && x86_64 && aws" }
steps {
run_project_tests('koji-osbuild', 'integration.sh')
}
post {
always {
preserve_logs('fedora33-revdep-koji-osbuild')
}
}
}
stage('EL8 Base') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "rhel8cloudbase && x86_64 && psi" }
environment {
TEST_TYPE = "base"
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
run_tests('base')
}
post {
always {
preserve_logs('rhel8-base')
}
}
}
stage('EL8 Integration') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "rhel8cloudbase && x86_64 && psi" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "rhel_8"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
VCENTER_CREDS = credentials('vmware-vcenter-credentials')
GOOGLE_APPLICATION_CREDENTIALS = credentials('gcp-credentials-osbuildci')
GCP_API_TEST_SHARE_ACCOUNT = credentials('gcp-credentials-share-account')
AZURE_CLIENT_ID = credentials('azure-client-id')
AZURE_CLIENT_SECRET = credentials('azure-client-secret')
AZURE_CREDS = credentials('azure')
}
steps {
run_tests('integration')
}
post {
always {
preserve_logs('rhel8-integration')
sh (
label: "Run cloud cleaner just in case something failed",
script: "schutzbot/run_cloud_cleaner.sh"
)
}
}
}
stage('EL8 OSTree') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "rhel8cloudbase && psi && x86_64" }
steps {
run_tests('ostree')
}
post {
always {
preserve_logs('rhel8-ostree')
}
}
}
stage('EL8: koji-osbuild') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "rhel8cloudbase && psi && x86_64" }
steps {
run_project_tests('koji-osbuild', 'integration.sh')
}
post {
always {
preserve_logs('rhel8-revdep-koji-osbuild')
}
}
}
stage('EL8 aarch64 Base') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "rhel8cloudbase && aarch64 && aws" }
environment {
TEST_TYPE = "base"
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
run_tests('base')
}
post {
always {
preserve_logs('el8-aarch64-base')
}
}
}
stage('EL8.4 Base') {
agent { label "rhel84cloudbase && x86_64 && psi" }
environment {
TEST_TYPE = "base"
}
steps {
run_tests('base')
}
post {
always {
preserve_logs('rhel84-base')
}
}
}
stage('EL8.4 Integration') {
agent { label "rhel84cloudbase && x86_64 && psi" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "rhel_84"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
VCENTER_CREDS = credentials('vmware-vcenter-credentials')
GOOGLE_APPLICATION_CREDENTIALS = credentials('gcp-credentials-osbuildci')
GCP_API_TEST_SHARE_ACCOUNT = credentials('gcp-credentials-share-account')
AZURE_CLIENT_ID = credentials('azure-client-id')
AZURE_CLIENT_SECRET = credentials('azure-client-secret')
AZURE_CREDS = credentials('azure')
}
steps {
run_tests('integration')
}
post {
always {
preserve_logs('rhel84-integration')
sh (
label: "Run cloud cleaner just in case something failed",
script: "schutzbot/run_cloud_cleaner.sh"
)
}
}
}
stage('EL8.4 OSTree') {
agent { label "rhel84cloudbase && psi && x86_64" }
steps {
run_tests('ostree')
}
post {
always {
preserve_logs('rhel84-ostree')
}
}
}
stage('EL8.4 New OSTree') {
agent { label "rhel84cloudbase && psi && x86_64" }
steps {
run_tests('ostree-ng')
}
post {
always {
preserve_logs('rhel84-ostree-ng')
}
}
}
stage('EL9.0 Integration') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "rhel84cloudbase && x86_64 && psi" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "rhel_90"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
GCP_API_TEST_SHARE_ACCOUNT = credentials('gcp-credentials-share-account')
AZURE_CLIENT_ID = credentials('azure-client-id')
AZURE_CLIENT_SECRET = credentials('azure-client-secret')
}
steps {
run_tests('integration')
}
post {
always {
preserve_logs('rhel84-integration')
sh (
label: "Run cloud cleaner just in case something failed",
script: "schutzbot/run_cloud_cleaner.sh"
)
}
}
}
stage('CS8 Base') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "cs8cloudbase && x86_64 && aws" }
environment {
TEST_TYPE = "base"
}
steps {
run_tests('base')
}
post {
always {
preserve_logs('cs8-base')
}
}
}
stage('CS8 Integration') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "cs8cloudbase && x86_64 && aws" }
environment {
TEST_TYPE = "integration"
DISTRO_CODE = "centos_8"
AWS_CREDS = credentials('aws-credentials-osbuildci')
AWS_IMAGE_TEST_CREDS = credentials('aws-credentials-osbuild-image-test')
AWS_API_TEST_SHARE_ACCOUNT = credentials('aws-credentials-share-account')
GOOGLE_APPLICATION_CREDENTIALS = credentials('gcp-credentials-osbuildci')
GCP_API_TEST_SHARE_ACCOUNT = credentials('gcp-credentials-share-account')
AZURE_CLIENT_ID = credentials('azure-client-id')
AZURE_CLIENT_SECRET = credentials('azure-client-secret')
AZURE_CREDS = credentials('azure')
}
steps {
run_tests('integration')
}
post {
always {
preserve_logs('cs8-integration')
sh (
label: "Run cloud cleaner just in case something failed",
script: "schutzbot/run_cloud_cleaner.sh"
)
}
}
}
stage('CS8 aarch64 Base') {
when {
beforeAgent true
expression {
return env.BUILD_CAUSE != 'cron';
}
}
agent { label "cs8cloudbase && aarch64 && aws" }
environment {
TEST_TYPE = "base"
}
steps {
run_tests('base')
}
post {
always {
preserve_logs('cs8-aarch64-base')
}
}
}
}
}
}
post {
success {
node('schutzbot') {
script {
if (env.BUILD_CAUSE == 'cron') {
unstash 'compose_id'
def composeId = readFile "${env.WORKSPACE}/COMPOSE_ID"
telegramSend """💚 CI passed for osbuild-composer ${composeId} ${env.BUILD_URL},
CC: [@atodorov_kiwi](tg://user?id=1047018883)
"""
} else if (env.BRANCH_NAME == 'main') {
telegramSend "💚 CI passed for osbuild-composer main branch ${env.BUILD_URL}"
}
}
}
}
unsuccessful {
node('schutzbot') {
script {
if (env.BUILD_CAUSE == 'cron') {
unstash 'compose_id'
def composeId = readFile "${env.WORKSPACE}/COMPOSE_ID"
telegramSend """💣 CI failed for osbuild-composer ${composeId} ${env.BUILD_URL},
CC: [@atodorov_kiwi](tg://user?id=1047018883)
"""
} else if (env.BRANCH_NAME == 'main') {
telegramSend "💣 CI failed for osbuild-composer main branch ${env.BUILD_URL}"
}
}
}
}
}
}
// Set up a function to hold the steps needed to run the tests so we don't
// need to copy/paste the same lines over and over above.
void run_tests(test_type) {
// unstash files but avoid failure if they don't exist
// b/c these files are available only for internal builds
try {
unstash 'rhel8internal'
unstash 'rhel8json'
unstash 'rhel8betajson'
} catch (err) {
echo "ERROR during unstash: ${err}"
}
// Get CI machine details.
sh (
label: "Get CI machine details",
script: "schutzbot/ci_details.sh"
)
// Deploy the Image Builder packages and services.
sh (
label: "Deploy",
script: "schutzbot/deploy.sh"
)
// Run the base tests.
if (test_type == 'base') {
sh (
label: "Base tests",
script: "/usr/libexec/tests/osbuild-composer/base_tests.sh"
)
sh (
label: "Regression tests",
script: "/usr/libexec/tests/osbuild-composer/regression.sh"
)
}
// Disabled. See commit log for details.
//if (test_type == 'image') {
// sh (
// label: "Image tests",
// script: "/usr/libexec/tests/osbuild-composer/image_tests.sh"
// )
//}
if (test_type == 'ostree') {
sh (
label: "OSTree tests",
script: "/usr/libexec/tests/osbuild-composer/ostree.sh"
)
}
if (test_type == 'ostree-ng') {
sh (
label: "New OSTree tests",
script: "/usr/libexec/tests/osbuild-composer/ostree-ng.sh"
)
}
if (test_type == 'integration') {
// Run Koji tests.
if (env.BUILD_CAUSE == 'cron') {
sh (
label: "Koji tests",
script: "/usr/libexec/tests/osbuild-composer/koji.sh"
)
// Run the libvirt test.
sh (
label: "Integration test: libvirt",
script: "/usr/libexec/tests/osbuild-composer/libvirt.sh"
)
// Run the AWS test.
sh (
label: "Integration test: AWS",
script: "/usr/libexec/tests/osbuild-composer/aws.sh"
)
// Run the Azure test.
sh (
label: "Integration test: AZURE",
script: "/usr/libexec/tests/osbuild-composer/azure.sh"
)
// Run the API test with AWS.
sh (
label: "Integration test: API (AWS)",
script: "/usr/libexec/tests/osbuild-composer/api.sh aws"
)
sh (
label: "Integration test: API (Azure)",
script: "/usr/libexec/tests/osbuild-composer/api.sh azure"
)
}
if (env.GOOGLE_APPLICATION_CREDENTIALS) {
// Run the API test with GCP.
sh (
label: "Integration test: API (GCP)",
script: "/usr/libexec/tests/osbuild-composer/api.sh gcp"
)
}
if (env.VCENTER_CREDS) {
// Run the VMWare test.
sh (
label: "Integration test: VMWare",
script: "/usr/libexec/tests/osbuild-composer/vmware.sh"
)
}
}
}
// Similar to run_tests, but with a more general signature: allows setting the
// project whose -tests package to install, and the name of the test to execute
void run_project_tests(project, test) {
sh "schutzbot/ci_details.sh"
sh "schutzbot/deploy.sh ${project}"
sh "/usr/libexec/tests/${project}/${test}"
}
// Move logs to a unique location and tell Jenkins to capture them on success
// or failure.
void preserve_logs(test_slug) {
// Save the systemd journal.
sh "journalctl --boot > systemd-journald.log"
// Make a directory for the log files and move the logs there.
sh "mkdir ${test_slug} && mv *.log *.jpg ${test_slug}/ || true"
// The workspace directory is not used everywhere, tests use temporary directory under /tmp/logs.
sh "mkdir -p ${test_slug} && find /tmp/logs/ -name '*.log' -exec mv {} ${test_slug}/ \\; || true"
// Artifact the repo file.
sh "mkdir -p ${test_slug} && cp /etc/yum.repos.d/osbuild*.repo ${test_slug}/ || true"
// Artifact the logs.
archiveArtifacts (
allowEmptyArchive: true,
artifacts: "${test_slug}/*.log,${test_slug}/*.jpg,${test_slug}/*.repo"
)
}
void detect_build_cause() {
def buildCause = currentBuild.getBuildCauses().get(0)
if ( buildCause.shortDescription == 'Started by timer' ) {
return "cron"
}
return
}