debian-forge-composer/.gitlab-ci.yml
2023-12-06 16:52:40 +01:00

759 lines
19 KiB
YAML

stages:
- init
- rpmbuild
- prepare-rhel-internal
- test
- finish
.base:
before_script:
- mkdir -p /tmp/artifacts
- schutzbot/ci_details.sh > /tmp/artifacts/ci-details-before-run.txt
- cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null
after_script:
- schutzbot/ci_details.sh > /tmp/artifacts/ci-details-after-run.txt || true
- schutzbot/unregister.sh || true
- schutzbot/update_github_status.sh update || true
- schutzbot/save_journal.sh || true
- schutzbot/upload_artifacts.sh
interruptible: true
retry: 1
tags:
- terraform
artifacts:
paths:
- "*.repo"
- COMPOSER_NVR
when: always
.terraform:
extends: .base
tags:
- terraform
.terraform/openstack:
extends: .base
tags:
- terraform/openstack
.terraform/gcp:
extends: .base
tags:
- terraform/gcp
init:
stage: init
interruptible: true
tags:
- shell
script:
- schutzbot/update_github_status.sh start
.build_rules:
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule" && $SKIP_CI == "false"'
- if: '$CI_PIPELINE_SOURCE != "schedule" && $SKIP_CI == "true"'
when: manual
.upstream_rules_all:
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule"'
.upstream_rules_x86_64:
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule" && $RUNNER =~ "/^.*(x86_64).*$/"'
.nightly_rules_all:
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-9.4-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "9"'
- if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-8.10-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "8"'
.nightly_rules_x86_64:
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-9.4-[^ga][\S]+/ && $RUNNER =~ "/^.*(x86_64).*$/" && $NIGHTLY == "true" && $RHEL_MAJOR == "9"'
- if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-8.10-[^ga][\S]+/ && $RUNNER =~ "/^.*(x86_64).*$/" && $NIGHTLY == "true" && $RHEL_MAJOR == "8"'
.RPM_RUNNERS_RHEL: &RPM_RUNNERS_RHEL
RUNNER:
- aws/rhel-8.10-nightly-x86_64
- aws/rhel-8.10-nightly-aarch64
- aws/rhel-9.4-nightly-x86_64
- aws/rhel-9.4-nightly-aarch64
INTERNAL_NETWORK: ["true"]
RPM:
stage: rpmbuild
extends: .terraform
rules:
- !reference [.build_rules, rules]
script:
- sh "schutzbot/mockbuild.sh"
parallel:
matrix:
- RUNNER:
- aws/fedora-37-x86_64
- aws/fedora-37-aarch64
- aws/fedora-38-x86_64
- aws/fedora-38-aarch64
- aws/fedora-40-x86_64
- aws/fedora-40-aarch64
- aws/rhel-8.4-ga-x86_64
- aws/rhel-8.4-ga-aarch64
- aws/rhel-8.8-ga-x86_64
- aws/rhel-8.8-ga-aarch64
- aws/rhel-9.2-ga-x86_64
- aws/rhel-9.2-ga-aarch64
- aws/centos-stream-8-x86_64
- aws/centos-stream-8-aarch64
- aws/centos-stream-9-x86_64
- aws/centos-stream-9-aarch64
- <<: *RPM_RUNNERS_RHEL
Build -tests RPM for RHEL:
stage: rpmbuild
extends: .terraform
rules:
- !reference [.nightly_rules_all, rules]
script:
- sh "schutzbot/mockbuild.sh"
interruptible: true
after_script:
- schutzbot/update_github_status.sh update
- schutzbot/save_journal.sh
parallel:
matrix:
- <<: *RPM_RUNNERS_RHEL
Container:
stage: rpmbuild
extends: .terraform
rules:
- !reference [.build_rules, rules]
script:
- sh "schutzbot/containerbuild.sh"
parallel:
matrix:
- RUNNER:
- aws/rhel-8.8-ga-x86_64
Packer:
stage: test
extends: .terraform
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule"'
script:
- tools/appsre-build-worker-packer.sh
variables:
RUNNER: aws/rhel-9.2-ga-x86_64
Prepare-rhel-internal:
stage: prepare-rhel-internal
extends: .terraform
rules:
- !reference [.nightly_rules_all, rules]
script:
- schutzbot/prepare-rhel-internal.sh
artifacts:
paths:
- rhel-${RHEL_MAJOR}.json
- rhel${RHEL_MAJOR}internal.repo
- COMPOSE_ID
parallel:
matrix:
- RUNNER:
# NOTE: 1 runner prepares for all arches b/c subsequent jobs download
# artifacts from all previous jobs and the last one wins
- aws/rhel-8.10-nightly-x86_64
- aws/rhel-9.4-nightly-x86_64
INTERNAL_NETWORK: ["true"]
Base:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules_all, rules]
- !reference [.nightly_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/base_tests.sh
parallel:
matrix:
- RUNNER:
- aws/fedora-37-x86_64
- aws/fedora-37-aarch64
- aws/fedora-38-x86_64
- aws/fedora-38-aarch64
- aws/rhel-8.4-ga-x86_64
- aws/rhel-8.4-ga-aarch64
- aws/rhel-8.8-ga-x86_64
- aws/rhel-8.8-ga-aarch64
- aws/rhel-9.2-ga-x86_64
- aws/rhel-9.2-ga-aarch64
- aws/centos-stream-8-x86_64
- aws/centos-stream-8-aarch64
- RUNNER:
- aws/rhel-8.10-nightly-x86_64
- aws/rhel-8.10-nightly-aarch64
- aws/rhel-9.4-nightly-x86_64
- aws/rhel-9.4-nightly-aarch64
- aws/centos-stream-9-x86_64
- aws/centos-stream-9-aarch64
INTERNAL_NETWORK: ["true"]
.regression:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules_all, rules]
- !reference [.nightly_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/${SCRIPT}
parallel:
matrix:
- RUNNER:
- aws/fedora-37-x86_64
- aws/fedora-37-aarch64
- aws/fedora-38-x86_64
- aws/fedora-38-aarch64
- aws/centos-stream-8-x86_64
- aws/centos-stream-8-aarch64
- RUNNER:
- aws/rhel-8.4-ga-x86_64
- aws/rhel-8.4-ga-aarch64
- aws/rhel-8.8-ga-x86_64
- aws/rhel-8.8-ga-aarch64
- aws/rhel-9.2-ga-x86_64
- aws/rhel-9.2-ga-aarch64
- aws/rhel-8.10-nightly-x86_64
- aws/rhel-8.10-nightly-aarch64
- aws/rhel-9.4-nightly-x86_64
- aws/rhel-9.4-nightly-aarch64
- aws/centos-stream-9-x86_64
- aws/centos-stream-9-aarch64
INTERNAL_NETWORK: ["true"]
regression-bigiso:
extends: .regression
rules:
# WHITELIST: Run only on x86_64 and rhel like systems
- if: $RUNNER =~ "/^.*(rhel-8.*x86_64|rhel-9.*x86_64|centos-stream-8.*x86_64|centos-stream-9.*x86_64).*$/" && $CI_PIPELINE_SOURCE != "schedule"
- !reference [.nightly_rules_x86_64, rules]
variables:
SCRIPT: regression-bigiso.sh
regression-composer-works-behind-satellite-fallback:
extends: .regression
rules:
# BLACKLIST: Skipped on subscribed RHEL machines
- if: $RUNNER !~ "/^.*(rhel-.*-ga|centos|fedora).*$/" && $CI_PIPELINE_SOURCE != "schedule"
- !reference [.nightly_rules_all, rules]
variables:
SCRIPT: regression-composer-works-behind-satellite-fallback.sh
regression-composer-works-behind-satellite:
extends: .regression
rules:
# BLACKLIST: Skipped on subscribed RHEL machines
- if: $RUNNER !~ "/^.*(rhel-.*-ga|centos|fedora).*$/" && $CI_PIPELINE_SOURCE != "schedule"
- !reference [.nightly_rules_all, rules]
variables:
SCRIPT: regression-composer-works-behind-satellite.sh
regression-excluded-dependency:
extends: .regression
rules:
# WHITELIST
- if: $RUNNER =~ "/^.*(rhel-8.*|rhel-9.*|centos-stream-8|centos-stream-9).*$/" && $CI_PIPELINE_SOURCE != "schedule"
- !reference [.nightly_rules_all, rules]
variables:
SCRIPT: regression-excluded-dependency.sh
regression-include-excluded-packages:
extends: .regression
rules:
# BLACKLIST: Skipped on fedora systems
- if: $RUNNER !~ "/^.*(fedora).*$/" && $CI_PIPELINE_SOURCE != "schedule"
- !reference [.nightly_rules_all, rules]
variables:
SCRIPT: regression-include-excluded-packages.sh
regression-old-worker-new-composer:
parallel:
matrix:
- RUNNER:
- aws/rhel-8.8-ga-x86_64
- aws/rhel-9.2-ga-x86_64
INTERNAL_NETWORK: ["true"]
extends: .regression
variables:
SCRIPT: regression-old-worker-new-composer.sh
regression-insecure-repo:
extends: .regression
rules:
# WHITELIST
- if: $RUNNER =~ "/^.*(rhel-*).*$/" && $CI_PIPELINE_SOURCE != "schedule"
- !reference [.nightly_rules_all, rules]
variables:
SCRIPT: regression-insecure-repo.sh
regression-no-explicit-rootfs-definition:
extends: .regression
rules:
# BLACKLIST: Skipped on fedora systems
- if: $RUNNER !~ "/^.*(fedora).*$/" && $CI_PIPELINE_SOURCE != "schedule"
- !reference [.nightly_rules_all, rules]
variables:
SCRIPT: regression-no-explicit-rootfs-definition.sh
Image Tests:
stage: test
extends: .terraform
rules:
- !reference [.nightly_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/image_tests.sh
parallel:
matrix:
- RUNNER:
- aws/rhel-8.4-ga-x86_64
- aws/rhel-8.4-ga-aarch64
- aws/rhel-8.10-nightly-x86_64
- aws/rhel-8.10-nightly-aarch64
- aws/rhel-9.4-nightly-x86_64
- aws/rhel-9.4-nightly-aarch64
INTERNAL_NETWORK: ["true"]
Trigger-rhel-edge-ci:
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule"'
stage: test
interruptible: true
tags:
- shell
script:
- 'curl -u "${SCHUTZBOT_LOGIN}" -X POST -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/osbuild/rhel-edge-ci/dispatches -d "{\"event_type\":\"osbuild-composer-ci\",\"client_payload\":{\"pr_number\":\"${CI_COMMIT_BRANCH}\"}}"'
.integration_base:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules_all, rules]
- !reference [.nightly_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/${SCRIPT}
.rhel_runners: &rhel_runners
RUNNER:
- aws/centos-stream-8-x86_64
- aws/rhel-8.4-ga-x86_64
- aws/rhel-8.8-ga-x86_64
- aws/rhel-9.2-ga-x86_64
- aws/rhel-8.10-nightly-x86_64
- aws/rhel-8.10-nightly-aarch64
- aws/rhel-9.4-nightly-x86_64
- aws/rhel-9.4-nightly-aarch64
- aws/centos-stream-9-x86_64
INTERNAL_NETWORK: ["true"]
.integration_rhel:
extends: .integration_base
parallel:
matrix:
- *rhel_runners
.fedora_runners: &fedora_runners
RUNNER:
- aws/fedora-37-x86_64
- aws/fedora-38-x86_64
.integration_fedora:
extends: .integration_base
parallel:
matrix:
- *fedora_runners
.integration:
extends: .integration_base
parallel:
matrix:
- *fedora_runners
- *rhel_runners
koji.sh (cloud upload):
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/koji.sh cloud-upload ${CLOUD_TARGET} ${IMAGE_TYPE}
parallel:
matrix:
- RUNNER:
# Brew workers use RHEL-8.7
- aws/rhel-8.8-ga-x86_64
INTERNAL_NETWORK: ["true"]
CLOUD_TARGET: aws
IMAGE_TYPE: aws-rhui
- RUNNER:
# Brew workers use RHEL-8.6
- aws/rhel-8.8-ga-x86_64
INTERNAL_NETWORK: ["true"]
CLOUD_TARGET: azure
IMAGE_TYPE: azure-rhui
- RUNNER:
# Brew workers use RHEL-8.6
- aws/rhel-8.8-ga-x86_64
INTERNAL_NETWORK: ["true"]
CLOUD_TARGET: gcp
IMAGE_TYPE: gcp-rhui
koji.sh (cloudapi):
extends: .integration
# Not supported in nightly pipelines
rules:
- !reference [.upstream_rules_all, rules]
variables:
SCRIPT: koji.sh
aws.sh:
extends: .integration
variables:
SCRIPT: aws.sh
oci.sh:
extends: .integration
rules:
# Run only on x86_64
- !reference [.upstream_rules_x86_64, rules]
variables:
SCRIPT: oci.sh
azure.sh:
extends: .integration
rules:
# Run only on x86_64
- !reference [.upstream_rules_x86_64, rules]
- !reference [.nightly_rules_x86_64, rules]
variables:
SCRIPT: azure.sh
azure.sh_hyperv_gen2:
extends: .integration
rules:
# Run only on x86_64
- !reference [.upstream_rules_x86_64, rules]
- !reference [.nightly_rules_x86_64, rules]
variables:
SCRIPT: azure_hyperv_gen2.sh
# The required GCE image type is not supported on Fedora
gcp.sh:
extends: .integration_rhel
rules:
- !reference [.upstream_rules_x86_64, rules]
- !reference [.nightly_rules_x86_64, rules]
variables:
SCRIPT: gcp.sh
vmware.sh_vmdk:
extends: .integration_rhel
rules:
# Run only on x86_64
- !reference [.upstream_rules_x86_64, rules]
- !reference [.nightly_rules_x86_64, rules]
variables:
SCRIPT: vmware.sh vmdk
vmware.sh_ova:
extends: .integration_rhel
rules:
# Run only on x86_64
- !reference [.upstream_rules_x86_64, rules]
- !reference [.nightly_rules_x86_64, rules]
variables:
SCRIPT: vmware.sh ova
filesystem.sh:
extends: .integration
variables:
SCRIPT: filesystem.sh
cross-distro.sh:
extends: .integration
variables:
SCRIPT: cross-distro.sh
.API_TESTS: &API_TESTS
IMAGE_TYPE:
- aws
- azure
- edge-commit
- gcp
- vsphere
- edge-commit generic.s3
- edge-container
- oci
API:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules_all, rules]
# note: cloud API is not supported for on-prem installations so
# don't run this test case for nightly trees
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/api.sh ${IMAGE_TYPE}
parallel:
matrix:
- <<: *API_TESTS
RUNNER:
- aws/rhel-8.8-ga-x86_64
- aws/rhel-9.2-ga-x86_64
- aws/rhel-8.10-nightly-x86_64
- aws/rhel-9.4-nightly-x86_64
INTERNAL_NETWORK: ["true"]
- IMAGE_TYPE: ["iot-commit"]
RUNNER:
- aws/fedora-37-x86_64
- aws/fedora-38-x86_64
- IMAGE_TYPE: ["aws"]
RUNNER:
- aws/rhel-8.8-ga-aarch64
- aws/rhel-9.2-ga-aarch64
INTERNAL_NETWORK: ["true"]
.libvirt_integration:
stage: test
extends: .terraform/gcp
rules:
- !reference [.upstream_rules_all, rules]
- !reference [.nightly_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/${SCRIPT}
parallel:
matrix:
- RUNNER:
- rhos-01/fedora-37-x86_64
- rhos-01/fedora-38-x86_64
- RUNNER:
- gcp/centos-stream-8-x86_64
- gcp/rhel-8.8-ga-x86_64
- gcp/rhel-9.2-ga-x86_64
- gcp/rhel-8.10-nightly-x86_64
- gcp/rhel-9.4-nightly-x86_64
- gcp/centos-stream-9-x86_64
INTERNAL_NETWORK: ["true"]
libvirt.sh:
extends: .libvirt_integration
variables:
SCRIPT: libvirt.sh
ubi-wsl.sh:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules_all, rules]
- !reference [.nightly_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/ubi-wsl.sh
variables:
RUNNER: aws/rhel-8.8-ga-x86_64
INTERNAL_NETWORK: "true"
.generic_s3:
extends: .libvirt_integration
rules:
# BLACKLIST
- if: $RUNNER !~ "/^.*(rhel-8.10|rhel-9.4).*$/" && $CI_PIPELINE_SOURCE != "schedule" && $NIGHTLY != "true"
- !reference [.nightly_rules_all, rules]
generic_s3_http.sh:
extends: .generic_s3
variables:
SCRIPT: generic_s3_http.sh
generic_s3_https_secure.sh:
extends: .generic_s3
variables:
SCRIPT: generic_s3_https_secure.sh
generic_s3_https_insecure.sh:
extends: .generic_s3
variables:
SCRIPT: generic_s3_https_insecure.sh
aws_s3.sh:
extends: .generic_s3
variables:
SCRIPT: aws_s3.sh
RHEL 9 on 8:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/koji.sh
variables:
RUNNER: aws/rhel-8.8-ga-x86_64
INTERNAL_NETWORK: "true"
DISTRO_CODE: rhel-91
Multi-tenancy:
stage: test
extends: .terraform
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule"'
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/multi-tenancy.sh
variables:
# only 8.7 GA b/c the Image Builder service runs on RHEL 8
RUNNER: aws/rhel-8.8-ga-x86_64
INTERNAL_NETWORK: "true"
OpenSCAP:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/oscap.sh
parallel:
matrix:
- RUNNER:
- rhos-01/rhel-8.10-nightly-x86_64
- rhos-01/rhel-9.4-nightly-x86_64
Upgrade:
stage: test
extends: .terraform/openstack
rules:
- !reference [.nightly_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/upgrade8to9.sh
variables:
RUNNER: rhos-01/rhel-9.4-nightly-x86_64-large
NIGHTLY_FAIL:
stage: finish
tags:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true"'
when: on_failure
script:
- schutzbot/slack_notification.sh FAILED ":big-sad:"
NIGHTLY_SUCCESS:
stage: finish
tags:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true"'
script:
- schutzbot/slack_notification.sh SUCCESS ":partymeow:"
Installer:
stage: test
extends: .terraform/openstack
rules:
- !reference [.upstream_rules_all, rules]
- !reference [.nightly_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/installers.sh
parallel:
matrix:
- RUNNER:
- rhos-01/rhel-8.10-nightly-x86_64
- rhos-01/rhel-9.4-nightly-x86_64
- rhos-01/centos-stream-9-x86_64
Manifest-diff:
stage: test
extends: .terraform
rules:
# don't run on main
- if: '$CI_COMMIT_BRANCH != "main"'
script:
- ./test/cases/diff-manifests.sh
variables:
RUNNER: aws/rhel-9.2-ga-x86_64
INTERNAL_NETWORK: "true"
GIT_STRATEGY: "clone"
GIT_CHECKOUT: "true"
GIT_DEPTH: 500
artifacts:
paths:
- manifests.diff
SonarQube:
stage: test
extends: .terraform
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule" && $CI_COMMIT_BRANCH == "main"'
script:
- schutzbot/sonarqube.sh
variables:
RUNNER: aws/centos-stream-8-x86_64
INTERNAL_NETWORK: "true"
GIT_DEPTH: 0
ContainerUpload:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/container-upload.sh
parallel:
matrix:
- RUNNER:
- aws/fedora-37-x86_64
- aws/fedora-38-x86_64
ContainerEmbedding:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules_all, rules]
- !reference [.nightly_rules_all, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/container-embedding.sh
parallel:
matrix:
- INTERNAL_NETWORK: "true"
RUNNER:
- aws/centos-stream-8-x86_64
- aws/centos-stream-9-x86_64
- aws/rhel-8.10-nightly-x86_64
- aws/rhel-8.10-nightly-aarch64
- aws/rhel-9.4-nightly-x86_64
- aws/rhel-9.4-nightly-aarch64
finish:
stage: finish
dependencies: []
tags:
- shell
script:
- schutzbot/update_github_status.sh finish