bee14bf392
jobimpl-osbuild --------------- Add GenericS3Creds to struct Add method to create AWS with Endpoint for Generic S3 (with its own credentials file) Move uploading to S3 and result handling to a separate method (along with the special VMDK handling) adjust the AWS S3 case to the new method Implement a new case for uploading to a generic S3 service awscloud -------- Add wrapper methods for endpoint support Set the endpoint to the AWS session Set s3ForcePathStyle to true if endpoint was set Target ------ Define a new target type for the GenericS3Target and Options Handle unmarshaling of the target options and result for the Generic S3 Weldr ----- Add support for only uploading to AWS S3 Define new structures for AWS S3 and Generic S3 (based on AWS S3) Handle unmarshaling of the providers settings' upload settings main ---- Add a section in the main config for the Generic S3 service for credentials If provided pass the credentials file name to the osbuild job implementation Upload Utility -------------- Add upload-generic-s3 utility Makefile ------ Do not fail if the bin directory already exists Tests ----- Add test cases for both AWS and a generic S3 server Add a generic s3_test.sh file for both test cases and add it to the tests RPM spec Adjust the libvirt test case script to support already created images GitLabCI - Extend the libvirt test case to include the two new tests
591 lines
14 KiB
YAML
591 lines
14 KiB
YAML
stages:
|
|
- init
|
|
- rpmbuild
|
|
- prepare-rhel-internal
|
|
- test
|
|
- cleanup
|
|
- finish
|
|
|
|
.base:
|
|
before_script:
|
|
- schutzbot/ci_details.sh > ci-details-before-run
|
|
after_script:
|
|
- schutzbot/ci_details.sh > ci-details-after-run
|
|
- schutzbot/update_github_status.sh update
|
|
- schutzbot/save_journal.sh
|
|
- schutzbot/run_cloud_cleaner.sh
|
|
interruptible: true
|
|
tags:
|
|
- terraform
|
|
artifacts:
|
|
paths:
|
|
- journal-log.gpg
|
|
- ci-artifacts
|
|
- ci-details-before-run
|
|
- ci-details-after-run
|
|
- "*.repo"
|
|
- COMPOSER_NVR
|
|
when: always
|
|
|
|
.terraform:
|
|
extends: .base
|
|
tags:
|
|
- terraform
|
|
|
|
.terraform/openstack:
|
|
extends: .base
|
|
tags:
|
|
- terraform/openstack
|
|
|
|
|
|
init:
|
|
stage: init
|
|
tags:
|
|
- shell
|
|
script:
|
|
- schutzbot/update_github_status.sh start
|
|
|
|
.build_rules:
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE != "schedule" && $SKIP_CI == "false"'
|
|
- if: '$CI_PIPELINE_SOURCE != "schedule" && $SKIP_CI == "true"'
|
|
when: manual
|
|
|
|
.upstream_rules:
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE != "schedule"'
|
|
|
|
.nightly_rules:
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-9.0-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "9"'
|
|
- if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-8.6-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "8"'
|
|
|
|
|
|
.RPM_RUNNERS_RHEL: &RPM_RUNNERS_RHEL
|
|
RUNNER:
|
|
- aws/rhel-8.6-nightly-x86_64
|
|
- aws/rhel-8.6-nightly-aarch64
|
|
- aws/rhel-9.0-nightly-x86_64
|
|
- aws/rhel-9.0-nightly-aarch64
|
|
- aws/centos-stream-9-x86_64
|
|
- aws/centos-stream-9-aarch64
|
|
|
|
INTERNAL_NETWORK: ["true"]
|
|
|
|
RPM:
|
|
stage: rpmbuild
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.build_rules, rules]
|
|
script:
|
|
- sh "schutzbot/mockbuild.sh"
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- aws/fedora-34-x86_64
|
|
- aws/fedora-34-aarch64
|
|
- aws/fedora-35-x86_64
|
|
- aws/fedora-35-aarch64
|
|
- aws/rhel-8.5-ga-x86_64
|
|
- aws/rhel-8.5-ga-aarch64
|
|
- aws/centos-stream-8-x86_64
|
|
- aws/centos-stream-8-aarch64
|
|
- <<: *RPM_RUNNERS_RHEL
|
|
|
|
Build -tests RPM for RHEL:
|
|
stage: rpmbuild
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- sh "schutzbot/mockbuild.sh"
|
|
interruptible: true
|
|
after_script:
|
|
- schutzbot/update_github_status.sh update
|
|
- schutzbot/save_journal.sh
|
|
parallel:
|
|
matrix:
|
|
- <<: *RPM_RUNNERS_RHEL
|
|
|
|
Container:
|
|
stage: rpmbuild
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.build_rules, rules]
|
|
script:
|
|
- sh "schutzbot/containerbuild.sh"
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- aws/rhel-8.5-ga-x86_64
|
|
|
|
Packer:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE != "schedule"'
|
|
script:
|
|
- tools/appsre-build-worker-packer.sh
|
|
variables:
|
|
RUNNER: aws/rhel-8.5-ga-x86_64
|
|
|
|
Prepare-rhel-internal:
|
|
stage: prepare-rhel-internal
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- schutzbot/prepare-rhel-internal.sh
|
|
artifacts:
|
|
paths:
|
|
- rhel-${RHEL_MAJOR}.json
|
|
- rhel${RHEL_MAJOR}internal.repo
|
|
- rhel-${RHEL_MAJOR}-beta.json
|
|
- COMPOSE_ID
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
# NOTE: 1 runner prepares for all arches b/c subsequent jobs download
|
|
# artifacts from all previous jobs and the last one wins
|
|
- aws/rhel-8.6-nightly-x86_64
|
|
- aws/rhel-9.0-nightly-x86_64
|
|
INTERNAL_NETWORK: ["true"]
|
|
|
|
Base:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.upstream_rules, rules]
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/base_tests.sh
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- aws/fedora-34-x86_64
|
|
- aws/fedora-34-aarch64
|
|
- aws/fedora-35-x86_64
|
|
- aws/fedora-35-aarch64
|
|
- aws/rhel-8.5-ga-x86_64
|
|
- aws/rhel-8.5-ga-aarch64
|
|
- aws/centos-stream-8-x86_64
|
|
- aws/centos-stream-8-aarch64
|
|
- RUNNER:
|
|
- aws/rhel-8.6-nightly-x86_64
|
|
- aws/rhel-8.6-nightly-aarch64
|
|
- aws/rhel-9.0-nightly-x86_64
|
|
- aws/rhel-9.0-nightly-aarch64
|
|
- aws/centos-stream-9-x86_64
|
|
- aws/centos-stream-9-aarch64
|
|
INTERNAL_NETWORK: ["true"]
|
|
|
|
Manifests:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.upstream_rules, rules]
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/manifest_tests.sh
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- aws/rhel-9.0-nightly-x86_64
|
|
INTERNAL_NETWORK: ["true"]
|
|
|
|
.regression:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.upstream_rules, rules]
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/${SCRIPT}
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- aws/fedora-34-x86_64
|
|
- aws/fedora-34-aarch64
|
|
- aws/fedora-35-x86_64
|
|
- aws/fedora-35-aarch64
|
|
- aws/centos-stream-8-x86_64
|
|
- aws/centos-stream-8-aarch64
|
|
- RUNNER:
|
|
- aws/rhel-8.5-ga-x86_64
|
|
- aws/rhel-8.5-ga-aarch64
|
|
- aws/rhel-8.6-nightly-x86_64
|
|
- aws/rhel-8.6-nightly-aarch64
|
|
- aws/rhel-9.0-nightly-x86_64
|
|
- aws/rhel-9.0-nightly-aarch64
|
|
- aws/centos-stream-9-x86_64
|
|
- aws/centos-stream-9-aarch64
|
|
INTERNAL_NETWORK: ["true"]
|
|
|
|
regression-bigiso:
|
|
extends: .regression
|
|
variables:
|
|
SCRIPT: regression-bigiso.sh
|
|
|
|
regression-composer-works-behind-satellite-fallback:
|
|
extends: .regression
|
|
variables:
|
|
SCRIPT: regression-composer-works-behind-satellite-fallback.sh
|
|
|
|
regression-composer-works-behind-satellite:
|
|
extends: .regression
|
|
variables:
|
|
SCRIPT: regression-composer-works-behind-satellite.sh
|
|
|
|
regression-excluded-dependency:
|
|
extends: .regression
|
|
variables:
|
|
SCRIPT: regression-excluded-dependency.sh
|
|
|
|
regression-include-excluded-packages:
|
|
extends: .regression
|
|
variables:
|
|
SCRIPT: regression-include-excluded-packages.sh
|
|
|
|
regression-old-worker-new-composer.sh:
|
|
extends: .regression
|
|
variables:
|
|
SCRIPT: regression-old-worker-new-composer.sh
|
|
|
|
Image Tests:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/image_tests.sh
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- aws/rhel-8.6-nightly-x86_64
|
|
- aws/rhel-8.6-nightly-aarch64
|
|
# See https://github.com/osbuild/osbuild-composer/issues/1819 and related issues
|
|
- aws/rhel-9.0-nightly-x86_64
|
|
- aws/rhel-9.0-nightly-aarch64
|
|
INTERNAL_NETWORK: ["true"]
|
|
|
|
Test Case Generation:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.upstream_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/generation.sh
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- aws/fedora-34-x86_64
|
|
INTERNAL_NETWORK: ["true"]
|
|
|
|
OSTree:
|
|
stage: test
|
|
extends: .terraform/openstack
|
|
rules:
|
|
- !reference [.upstream_rules, rules]
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/ostree.sh
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- rhos-01/fedora-35-x86_64
|
|
- rhos-01/rhel-8.5-ga-x86_64
|
|
- rhos-01/rhel-8.6-nightly-x86_64-large
|
|
- rhos-01/rhel-9.0-nightly-x86_64
|
|
- rhos-01/centos-stream-8-x86_64
|
|
- rhos-01/centos-stream-9-x86_64
|
|
|
|
New OSTree:
|
|
stage: test
|
|
extends: OSTree
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/ostree-ng.sh
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- rhos-01/rhel-8.6-nightly-x86_64-large
|
|
- rhos-01/rhel-9.0-nightly-x86_64-large
|
|
- rhos-01/centos-stream-8-x86_64-large
|
|
- rhos-01/centos-stream-9-x86_64-large
|
|
|
|
OSTree simplified installer:
|
|
stage: test
|
|
extends: OSTree
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/ostree-simplified-installer.sh
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- rhos-01/rhel-8.6-nightly-x86_64-large
|
|
- rhos-01/centos-stream-8-x86_64-large
|
|
- rhos-01/rhel-9.0-nightly-x86_64-large
|
|
- rhos-01/centos-stream-9-x86_64-large
|
|
|
|
OSTree raw image:
|
|
stage: test
|
|
extends: OSTree
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/ostree-raw-image.sh
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- rhos-01/rhel-8.6-nightly-x86_64-large
|
|
- rhos-01/centos-stream-8-x86_64-large
|
|
- rhos-01/rhel-9.0-nightly-x86_64-large
|
|
- rhos-01/centos-stream-9-x86_64-large
|
|
|
|
Rebase OSTree:
|
|
stage: test
|
|
extends: OSTree
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/ostree-rebase.sh
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- rhos-01/rhel-8.6-nightly-x86_64-large
|
|
- rhos-01/rhel-9.0-nightly-x86_64-large
|
|
- rhos-01/centos-stream-8-x86_64-large
|
|
- rhos-01/centos-stream-9-x86_64-large
|
|
|
|
.integration:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.upstream_rules, rules]
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/${SCRIPT}
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- aws/fedora-34-x86_64
|
|
- aws/fedora-35-x86_64
|
|
- aws/centos-stream-8-x86_64
|
|
- aws/rhel-8.5-ga-x86_64
|
|
- aws/rhel-8.6-nightly-x86_64
|
|
- aws/rhel-9.0-nightly-x86_64
|
|
- aws/centos-stream-9-x86_64
|
|
INTERNAL_NETWORK: ["true"]
|
|
|
|
koji.sh:
|
|
extends: .integration
|
|
variables:
|
|
SCRIPT: koji.sh
|
|
|
|
aws.sh:
|
|
extends: .integration
|
|
variables:
|
|
SCRIPT: aws.sh
|
|
|
|
azure.sh:
|
|
extends: .integration
|
|
variables:
|
|
SCRIPT: azure.sh
|
|
|
|
vmware.sh:
|
|
extends: .integration
|
|
variables:
|
|
SCRIPT: vmware.sh
|
|
|
|
filesystem.sh:
|
|
extends: .integration
|
|
variables:
|
|
SCRIPT: filesystem.sh
|
|
|
|
cross-distro.sh:
|
|
extends: .integration
|
|
variables:
|
|
SCRIPT: cross-distro.sh
|
|
|
|
.API_TESTS: &API_TESTS
|
|
IMAGE_TYPE:
|
|
- aws
|
|
- azure
|
|
- edge-commit
|
|
- gcp
|
|
- vsphere
|
|
|
|
API:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.upstream_rules, rules]
|
|
# note: cloud API is not supported for on-prem installations so
|
|
# don't run this test case for nightly trees
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/api.sh ${IMAGE_TYPE}
|
|
parallel:
|
|
matrix:
|
|
- <<: *API_TESTS
|
|
RUNNER:
|
|
- aws/rhel-8.5-ga-x86_64
|
|
- aws/rhel-8.6-nightly-x86_64
|
|
- aws/rhel-9.0-nightly-x86_64
|
|
INTERNAL_NETWORK: ["true"]
|
|
- IMAGE_TYPE: ["edge-commit"]
|
|
RUNNER:
|
|
- aws/fedora-35-x86_64
|
|
|
|
.libvirt_integration:
|
|
stage: test
|
|
extends: .terraform/openstack
|
|
rules:
|
|
- !reference [.upstream_rules, rules]
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/${SCRIPT}
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- rhos-01/centos-stream-8-x86_64
|
|
- rhos-01/rhel-8.5-ga-x86_64
|
|
- rhos-01/rhel-8.6-nightly-x86_64
|
|
- rhos-01/rhel-9.0-nightly-x86_64
|
|
- rhos-01/centos-stream-9-x86_64
|
|
|
|
libvirt.sh:
|
|
extends: .libvirt_integration
|
|
variables:
|
|
SCRIPT: libvirt.sh
|
|
|
|
generic_s3.sh:
|
|
extends: .libvirt_integration
|
|
variables:
|
|
SCRIPT: generic_s3.sh
|
|
|
|
aws_s3.sh:
|
|
extends: .libvirt_integration
|
|
variables:
|
|
SCRIPT: aws_s3.sh
|
|
|
|
RHEL 9 on 8:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- !reference [.upstream_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/koji.sh
|
|
variables:
|
|
RUNNER: aws/rhel-8.5-ga-x86_64
|
|
INTERNAL_NETWORK: "true"
|
|
DISTRO_CODE: rhel-90
|
|
|
|
Multi-tenancy:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE != "schedule"'
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/multi-tenancy.sh
|
|
variables:
|
|
RUNNER: aws/rhel-8.5-ga-x86_64
|
|
INTERNAL_NETWORK: "true"
|
|
|
|
Upgrade:
|
|
stage: test
|
|
extends: .terraform/openstack
|
|
rules:
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/upgrade8to9.sh
|
|
variables:
|
|
RUNNER: rhos-01/rhel-9.0-nightly-x86_64
|
|
artifacts:
|
|
paths:
|
|
- journal-log.gpg
|
|
- ci-artifacts
|
|
- install_console.log
|
|
- upgrade_console.log
|
|
- leapp-preupgrade.log
|
|
- leapp-upgrade.log
|
|
- leapp-report.txt
|
|
- osbuild-upgrade8to9.log
|
|
- osbuild-upgrade8to9.json
|
|
when: always
|
|
|
|
NIGHTLY_FAIL:
|
|
stage: finish
|
|
tags:
|
|
- shell
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true"'
|
|
when: on_failure
|
|
script:
|
|
- schutzbot/slack_notification.sh FAILED ":big-sad:"
|
|
|
|
NIGHTLY_SUCCESS:
|
|
stage: finish
|
|
tags:
|
|
- shell
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true"'
|
|
script:
|
|
- schutzbot/slack_notification.sh SUCCESS ":partymeow:"
|
|
|
|
Installer:
|
|
stage: test
|
|
extends: .terraform/openstack
|
|
rules:
|
|
- !reference [.upstream_rules, rules]
|
|
- !reference [.nightly_rules, rules]
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- /usr/libexec/tests/osbuild-composer/installers.sh
|
|
parallel:
|
|
matrix:
|
|
- RUNNER:
|
|
- rhos-01/rhel-8.6-nightly-x86_64
|
|
- rhos-01/rhel-9.0-nightly-x86_64
|
|
- rhos-01/centos-stream-9-x86_64
|
|
|
|
SCHEDULED_CLOUD_CLEANER:
|
|
stage: cleanup
|
|
tags:
|
|
- terraform
|
|
variables:
|
|
RUNNER: aws/centos-stream-8-x86_64
|
|
INTERNAL_NETWORK: "true"
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "schedule" && $CLEANUP == "true"'
|
|
script:
|
|
- schutzbot/deploy.sh
|
|
- schutzbot/scheduled_cloud_cleaner.sh
|
|
|
|
SonarQube:
|
|
stage: test
|
|
extends: .terraform
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE != "schedule" && $CI_COMMIT_BRANCH == "main"'
|
|
script:
|
|
- schutzbot/sonarqube.sh
|
|
variables:
|
|
RUNNER: aws/centos-stream-8-x86_64
|
|
INTERNAL_NETWORK: "true"
|
|
GIT_DEPTH: 0
|
|
|
|
finish:
|
|
stage: finish
|
|
tags:
|
|
- shell
|
|
script:
|
|
- schutzbot/update_github_status.sh finish
|