debian-forge-composer/.gitlab-ci.yml

stages:
  - init
  - rpmbuild
  - prepare-rhel-internal
  - test
  - cleanup
  - finish

.base:
  before_script:
    - schutzbot/ci_details.sh > ci-details-before-run
    - cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null
  after_script:
    - schutzbot/ci_details.sh > ci-details-after-run
    - schutzbot/update_github_status.sh update
    - schutzbot/save_journal.sh
    - schutzbot/run_cloud_cleaner.sh
  interruptible: true
  tags:
    - terraform
  artifacts:
    paths:
      - journal-log.gpg
      - ci-artifacts
      - ci-details-before-run
      - ci-details-after-run
      - "*.repo"
      - COMPOSER_NVR
    when: always

.terraform:
  extends: .base
  tags:
    - terraform

.terraform/openstack:
  extends: .base
  tags:
    - terraform/openstack


init:
  stage: init
  tags:
    - shell
  script:
    - schutzbot/update_github_status.sh start

.build_rules:
  rules:
    - if: '$CI_PIPELINE_SOURCE != "schedule" && $SKIP_CI == "false"'
    - if: '$CI_PIPELINE_SOURCE != "schedule" && $SKIP_CI == "true"'
      when: manual

.upstream_rules:
  rules:
    - if: '$CI_PIPELINE_SOURCE != "schedule"'

.nightly_rules:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-9.0-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "9"'
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-8.6-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "8"'


.RPM_RUNNERS_RHEL: &RPM_RUNNERS_RHEL
  RUNNER:
    - aws/rhel-8.6-nightly-x86_64
    - aws/rhel-8.6-nightly-aarch64
    - aws/rhel-9.0-nightly-x86_64
    - aws/rhel-9.0-nightly-aarch64
    - aws/centos-stream-9-x86_64
    - aws/centos-stream-9-aarch64

  INTERNAL_NETWORK: ["true"]

RPM:
  stage: rpmbuild
  extends: .terraform
  rules:
    - !reference [.build_rules, rules]
  script:
    - sh "schutzbot/mockbuild.sh"
  parallel:
    matrix:
      - RUNNER:
          - aws/fedora-34-x86_64
          - aws/fedora-34-aarch64
          - aws/fedora-35-x86_64
          - aws/fedora-35-aarch64
          - aws/rhel-8.5-ga-x86_64
          - aws/rhel-8.5-ga-aarch64
          - aws/centos-stream-8-x86_64
          - aws/centos-stream-8-aarch64
      - <<: *RPM_RUNNERS_RHEL

Build -tests RPM for RHEL:
  stage: rpmbuild
  extends: .terraform
  rules:
    - !reference [.nightly_rules, rules]
  script:
    - sh "schutzbot/mockbuild.sh"
  interruptible: true
  after_script:
    - schutzbot/update_github_status.sh update
    - schutzbot/save_journal.sh
  parallel:
    matrix:
      - <<: *RPM_RUNNERS_RHEL

Container:
  stage: rpmbuild
  extends: .terraform
  rules:
    - !reference [.build_rules, rules]
  script:
    - sh "schutzbot/containerbuild.sh"
  parallel:
    matrix:
      - RUNNER:
          - aws/rhel-8.5-ga-x86_64

Packer:
  stage: test
  extends: .terraform
  rules:
    - if: '$CI_PIPELINE_SOURCE != "schedule"'
  script:
    - tools/appsre-build-worker-packer.sh
  variables:
    RUNNER: aws/rhel-8.5-ga-x86_64

Prepare-rhel-internal:
  stage: prepare-rhel-internal
  extends: .terraform
  rules:
    - !reference [.nightly_rules, rules]
  script:
    - schutzbot/prepare-rhel-internal.sh
  artifacts:
    paths:
      - rhel-${RHEL_MAJOR}.json
      - rhel${RHEL_MAJOR}internal.repo
      - rhel-${RHEL_MAJOR}-beta.json
      - COMPOSE_ID
  parallel:
    matrix:
      - RUNNER:
          # NOTE: 1 runner prepares for all arches b/c subsequent jobs download
          # artifacts from all previous jobs and the last one wins
          - aws/rhel-8.6-nightly-x86_64
          - aws/rhel-9.0-nightly-x86_64
        INTERNAL_NETWORK: ["true"]

Base:
  stage: test
  extends: .terraform
  rules:
    - !reference [.upstream_rules, rules]
    - !reference [.nightly_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/base_tests.sh
  parallel:
    matrix:
      - RUNNER:
          - aws/fedora-34-x86_64
          - aws/fedora-34-aarch64
          - aws/fedora-35-x86_64
          - aws/fedora-35-aarch64
          - aws/rhel-8.5-ga-x86_64
          - aws/rhel-8.5-ga-aarch64
          - aws/centos-stream-8-x86_64
          - aws/centos-stream-8-aarch64
      - RUNNER:
          - aws/rhel-8.6-nightly-x86_64
          - aws/rhel-8.6-nightly-aarch64
          - aws/rhel-9.0-nightly-x86_64
          - aws/rhel-9.0-nightly-aarch64
          - aws/centos-stream-9-x86_64
          - aws/centos-stream-9-aarch64
        INTERNAL_NETWORK: ["true"]

Manifests:
  stage: test
  extends: .terraform
  rules:
    - !reference [.upstream_rules, rules]
    - !reference [.nightly_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/manifest_tests.sh
  parallel:
    matrix:
      - RUNNER:
          - aws/rhel-9.0-nightly-x86_64
        INTERNAL_NETWORK: ["true"]

.regression:
  stage: test
  extends: .terraform
  rules:
    - !reference [.upstream_rules, rules]
    - !reference [.nightly_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/${SCRIPT}
  parallel:
    matrix:
      - RUNNER:
          - aws/fedora-34-x86_64
          - aws/fedora-34-aarch64
          - aws/fedora-35-x86_64
          - aws/fedora-35-aarch64
          - aws/centos-stream-8-x86_64
          - aws/centos-stream-8-aarch64
      - RUNNER:
          - aws/rhel-8.5-ga-x86_64
          - aws/rhel-8.5-ga-aarch64
          - aws/rhel-8.6-nightly-x86_64
          - aws/rhel-8.6-nightly-aarch64
          - aws/rhel-9.0-nightly-x86_64
          - aws/rhel-9.0-nightly-aarch64
          - aws/centos-stream-9-x86_64
          - aws/centos-stream-9-aarch64
        INTERNAL_NETWORK: ["true"]

regression-bigiso:
  extends: .regression
  variables:
    SCRIPT: regression-bigiso.sh

regression-composer-works-behind-satellite-fallback:
  extends: .regression
  variables:
    SCRIPT: regression-composer-works-behind-satellite-fallback.sh

regression-composer-works-behind-satellite:
  extends: .regression
  variables:
    SCRIPT: regression-composer-works-behind-satellite.sh

regression-excluded-dependency:
  extends: .regression
  variables:
    SCRIPT: regression-excluded-dependency.sh

regression-include-excluded-packages:
  extends: .regression
  variables:
    SCRIPT: regression-include-excluded-packages.sh

regression-old-worker-new-composer:
  extends: .regression
  variables:
    SCRIPT: regression-old-worker-new-composer.sh

regression-no-explicit-rootfs-definition:
  extends: .regression
  variables:
    SCRIPT: regression-no-explicit-rootfs-definition.sh

Image Tests:
  stage: test
  extends: .terraform
  rules:
    - !reference [.nightly_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/image_tests.sh
  parallel:
    matrix:
      - RUNNER:
          - aws/rhel-8.6-nightly-x86_64
          - aws/rhel-8.6-nightly-aarch64
          # See https://github.com/osbuild/osbuild-composer/issues/1819 and related issues
          - aws/rhel-9.0-nightly-x86_64
          - aws/rhel-9.0-nightly-aarch64
        INTERNAL_NETWORK: ["true"]

Test Case Generation:
  stage: test
  extends: .terraform
  rules:
    - !reference [.upstream_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/generation.sh
  parallel:
    matrix:
      - RUNNER:
          - aws/fedora-34-x86_64
        INTERNAL_NETWORK: ["true"]

OSTree:
  stage: test
  extends: .terraform/openstack
  rules:
    - !reference [.upstream_rules, rules]
    - !reference [.nightly_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/ostree.sh
  parallel:
    matrix:
      - RUNNER:
          - rhos-01/fedora-35-x86_64
          - rhos-01/rhel-8.5-ga-x86_64
          - rhos-01/rhel-8.6-nightly-x86_64-large
          - rhos-01/rhel-9.0-nightly-x86_64
          - rhos-01/centos-stream-8-x86_64
          - rhos-01/centos-stream-9-x86_64

New OSTree:
  stage: test
  extends: OSTree
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/ostree-ng.sh
  parallel:
    matrix:
      - RUNNER:
          - rhos-01/rhel-8.6-nightly-x86_64-large
          - rhos-01/rhel-9.0-nightly-x86_64-large
          - rhos-01/centos-stream-8-x86_64-large
          - rhos-01/centos-stream-9-x86_64-large

OSTree simplified installer:
  stage: test
  extends: OSTree
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/ostree-simplified-installer.sh
  parallel:
    matrix:
      - RUNNER:
          - rhos-01/rhel-8.6-nightly-x86_64-large
          - rhos-01/centos-stream-8-x86_64-large
          - rhos-01/rhel-9.0-nightly-x86_64-large
          - rhos-01/centos-stream-9-x86_64-large

OSTree raw image:
  stage: test
  extends: OSTree
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/ostree-raw-image.sh
  parallel:
    matrix:
      - RUNNER:
          - rhos-01/rhel-8.6-nightly-x86_64-large
          - rhos-01/centos-stream-8-x86_64-large
          - rhos-01/rhel-9.0-nightly-x86_64-large
          - rhos-01/centos-stream-9-x86_64-large

Rebase OSTree:
  stage: test
  extends: OSTree
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/ostree-rebase.sh
  parallel:
    matrix:
      - RUNNER:
          - rhos-01/rhel-8.6-nightly-x86_64-large
          - rhos-01/rhel-9.0-nightly-x86_64-large
          - rhos-01/centos-stream-8-x86_64-large
          - rhos-01/centos-stream-9-x86_64-large

.integration_base:
  stage: test
  extends: .terraform
  rules:
    - !reference [.upstream_rules, rules]
    - !reference [.nightly_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/${SCRIPT}

.rhel_runners: &rhel_runners
    RUNNER:
      - aws/centos-stream-8-x86_64
      - aws/rhel-8.5-ga-x86_64
      - aws/rhel-8.6-nightly-x86_64
      - aws/rhel-9.0-nightly-x86_64
      - aws/centos-stream-9-x86_64
    INTERNAL_NETWORK: ["true"]

.integration_rhel:
  extends: .integration_base
  parallel:
    matrix:
      - *rhel_runners

.fedora_runners: &fedora_runners
    RUNNER:
      - aws/fedora-34-x86_64
      - aws/fedora-35-x86_64

.integration_fedora:
  extends: .integration_base
  parallel:
    matrix:
      - *fedora_runners

.integration:
  extends: .integration_base
  parallel:
    matrix:
      - *fedora_runners
      - *rhel_runners

koji.sh:
  extends: .integration
  variables:
    SCRIPT: koji.sh

aws.sh:
  extends: .integration
  variables:
    SCRIPT: aws.sh

azure.sh:
  extends: .integration
  variables:
    SCRIPT: azure.sh

# The required GCE image type is not supported on Fedora
gcp.sh:
  extends: .integration_rhel
  variables:
    SCRIPT: gcp.sh

vmware.sh:
  extends: .integration
  variables:
    SCRIPT: vmware.sh

filesystem.sh:
  extends: .integration
  variables:
    SCRIPT: filesystem.sh

cross-distro.sh:
  extends: .integration
  variables:
    SCRIPT: cross-distro.sh

.API_TESTS: &API_TESTS
  IMAGE_TYPE:
    - aws
    - azure
    - edge-commit
    - gcp
    - vsphere

API:
  stage: test
  extends: .terraform
  rules:
    - !reference [.upstream_rules, rules]
    # note: cloud API is not supported for on-prem installations so
    # don't run this test case for nightly trees
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/api.sh ${IMAGE_TYPE}
  parallel:
    matrix:
      - <<: *API_TESTS
        RUNNER:
          - aws/rhel-8.5-ga-x86_64
          - aws/rhel-8.6-nightly-x86_64
          - aws/rhel-9.0-nightly-x86_64
        INTERNAL_NETWORK: ["true"]
      - IMAGE_TYPE: ["edge-commit"]
        RUNNER:
          - aws/fedora-35-x86_64

.libvirt_integration:
  stage: test
  extends: .terraform/openstack
  rules:
    - !reference [.upstream_rules, rules]
    - !reference [.nightly_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/${SCRIPT}
  parallel:
    matrix:
      - RUNNER:
          - rhos-01/centos-stream-8-x86_64
          - rhos-01/rhel-8.5-ga-x86_64
          - rhos-01/rhel-8.6-nightly-x86_64
          - rhos-01/rhel-9.0-nightly-x86_64
          - rhos-01/centos-stream-9-x86_64

libvirt.sh:
  extends: .libvirt_integration
  variables:
    SCRIPT: libvirt.sh

generic_s3.sh:
  extends: .libvirt_integration
  variables:
    SCRIPT: generic_s3.sh

aws_s3.sh:
  extends: .libvirt_integration
  variables:
    SCRIPT: aws_s3.sh

RHEL 9 on 8:
  stage: test
  extends: .terraform
  rules:
    - !reference [.upstream_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/koji.sh
  variables:
    RUNNER: aws/rhel-8.5-ga-x86_64
    INTERNAL_NETWORK: "true"
    DISTRO_CODE: rhel-90

Multi-tenancy:
  stage: test
  extends: .terraform
  rules:
    - if: '$CI_PIPELINE_SOURCE != "schedule"'
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/multi-tenancy.sh
  variables:
    RUNNER: aws/rhel-8.5-ga-x86_64
    INTERNAL_NETWORK: "true"

Upgrade:
  stage: test
  extends: .terraform/openstack
  rules:
    - !reference [.nightly_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/upgrade8to9.sh
  variables:
    RUNNER: rhos-01/rhel-9.0-nightly-x86_64
  artifacts:
    paths:
      - journal-log.gpg
      - ci-artifacts
      - install_console.log
      - upgrade_console.log
      - leapp-preupgrade.log
      - leapp-upgrade.log
      - leapp-report.txt
      - osbuild-upgrade8to9.log
      - osbuild-upgrade8to9.json
    when: always

NIGHTLY_FAIL:
  stage: finish
  tags:
    - shell
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true"'
      when: on_failure
  script:
    - schutzbot/slack_notification.sh FAILED ":big-sad:"

NIGHTLY_SUCCESS:
  stage: finish
  tags:
    - shell
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true"'
  script:
    - schutzbot/slack_notification.sh SUCCESS ":partymeow:"

Installer:
  stage: test
  extends: .terraform/openstack
  rules:
    - !reference [.upstream_rules, rules]
    - !reference [.nightly_rules, rules]
  script:
    - schutzbot/deploy.sh
    - /usr/libexec/tests/osbuild-composer/installers.sh
  parallel:
    matrix:
      - RUNNER:
          - rhos-01/rhel-8.6-nightly-x86_64
          - rhos-01/rhel-9.0-nightly-x86_64
          - rhos-01/centos-stream-9-x86_64

SCHEDULED_CLOUD_CLEANER:
  stage: cleanup
  tags:
    - terraform
  variables:
    RUNNER: aws/centos-stream-8-x86_64
    INTERNAL_NETWORK: "true"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CLEANUP == "true"'
  script:
    - schutzbot/deploy.sh
    - schutzbot/scheduled_cloud_cleaner.sh

SonarQube:
  stage: test
  extends: .terraform
  rules:
    - if: '$CI_PIPELINE_SOURCE != "schedule" && $CI_COMMIT_BRANCH == "main"'
  script:
    - schutzbot/sonarqube.sh
  variables:
    RUNNER: aws/centos-stream-8-x86_64
    INTERNAL_NETWORK: "true"
    GIT_DEPTH: 0

finish:
  stage: finish
  dependencies: []
  tags:
    - shell
  script:
    - schutzbot/update_github_status.sh finish