debian-forge-composer/.gitlab-ci.yml
Ygal Blum feb357e538 Support Generic S3 upload in Composer API
Use case
--------
If Endpoint is not set and Region is - upload to AWS S3
If both the Endpoint and Region are set - upload the Generic S3 via Weldr API
If neither the Endpoint and Region are set - upload the Generic S3 via Composer API (use configuration)

jobimpl-osbuild
---------------
Add configuration fields for Generic S3 upload
Support S3 upload requests coming from Weldr or Composer API to either AWS or Generic S3
Weldr API for Generic S3 requires that all connection parameters but the credentials be passed in the API call
Composer API for Generic S3 requires that all conneciton parameters are taken from the configuration
Adjust to the consolidation in Target and UploadOptions

Target and UploadOptions
------------------------
Add the fields that were specific to the Generic S3 structures to the AWS S3 one
Remove the structures for Generic S3 and always use the AWS S3 ones

Worker Main
-----------
Add Endpoint, Region, Bucket, CABundle and SkipSSLVerification to the configuration structure
Pass the values to the Server

Weldr API
---------
Keep the generic.s3 provider name to maintain the API, but unmarshel into awsS3UploadSettings

tests - api.sh
--------------
Allow the caller to specifiy either AWS or Generic S3 upload targets for specific image types
Implement the pieces required for testing upload to a Generic S3 service
In some cases generalize the AWS S3 functions for reuse

GitLab CI
---------
Add test case for api.sh tests with edge-commit and generic S3
2022-06-02 16:12:53 +03:00

634 lines
15 KiB
YAML

stages:
- init
- rpmbuild
- prepare-rhel-internal
- test
- cleanup
- finish
.base:
before_script:
- schutzbot/ci_details.sh > ci-details-before-run
- cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null
after_script:
- schutzbot/ci_details.sh > ci-details-after-run || true
- schutzbot/update_github_status.sh update || true
- schutzbot/save_journal.sh || true
- schutzbot/run_cloud_cleaner.sh || true
interruptible: true
tags:
- terraform
artifacts:
paths:
- journal-log.gpg
- ci-artifacts
- ci-details-before-run
- ci-details-after-run
- "*.repo"
- COMPOSER_NVR
when: always
.terraform:
extends: .base
tags:
- terraform
.terraform/openstack:
extends: .base
tags:
- terraform/openstack
init:
stage: init
tags:
- shell
script:
- schutzbot/update_github_status.sh start
.build_rules:
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule" && $SKIP_CI == "false"'
- if: '$CI_PIPELINE_SOURCE != "schedule" && $SKIP_CI == "true"'
when: manual
.upstream_rules:
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule"'
.nightly_rules:
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-9.0-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "9"'
- if: '$CI_PIPELINE_SOURCE == "schedule" && $RUNNER =~ /[\S]+rhel-8.6-[^ga][\S]+/ && $NIGHTLY == "true" && $RHEL_MAJOR == "8"'
.RPM_RUNNERS_RHEL: &RPM_RUNNERS_RHEL
RUNNER:
- aws/rhel-8.6-nightly-x86_64
- aws/rhel-8.6-nightly-aarch64
- aws/rhel-9.0-nightly-x86_64
- aws/rhel-9.0-nightly-aarch64
- aws/centos-stream-9-x86_64
- aws/centos-stream-9-aarch64
INTERNAL_NETWORK: ["true"]
RPM:
stage: rpmbuild
extends: .terraform
rules:
- !reference [.build_rules, rules]
script:
- sh "schutzbot/mockbuild.sh"
parallel:
matrix:
- RUNNER:
- aws/fedora-34-x86_64
- aws/fedora-34-aarch64
- aws/fedora-35-x86_64
- aws/fedora-35-aarch64
- aws/rhel-8.5-ga-x86_64
- aws/rhel-8.5-ga-aarch64
- aws/centos-stream-8-x86_64
- aws/centos-stream-8-aarch64
- <<: *RPM_RUNNERS_RHEL
Build -tests RPM for RHEL:
stage: rpmbuild
extends: .terraform
rules:
- !reference [.nightly_rules, rules]
script:
- sh "schutzbot/mockbuild.sh"
interruptible: true
after_script:
- schutzbot/update_github_status.sh update
- schutzbot/save_journal.sh
parallel:
matrix:
- <<: *RPM_RUNNERS_RHEL
Container:
stage: rpmbuild
extends: .terraform
rules:
- !reference [.build_rules, rules]
script:
- sh "schutzbot/containerbuild.sh"
parallel:
matrix:
- RUNNER:
- aws/rhel-8.5-ga-x86_64
Packer:
stage: test
extends: .terraform
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule"'
script:
- tools/appsre-build-worker-packer.sh
variables:
RUNNER: aws/rhel-8.5-ga-x86_64
Prepare-rhel-internal:
stage: prepare-rhel-internal
extends: .terraform
rules:
- !reference [.nightly_rules, rules]
script:
- schutzbot/prepare-rhel-internal.sh
artifacts:
paths:
- rhel-${RHEL_MAJOR}.json
- rhel${RHEL_MAJOR}internal.repo
- rhel-${RHEL_MAJOR}-beta.json
- COMPOSE_ID
parallel:
matrix:
- RUNNER:
# NOTE: 1 runner prepares for all arches b/c subsequent jobs download
# artifacts from all previous jobs and the last one wins
- aws/rhel-8.6-nightly-x86_64
- aws/rhel-9.0-nightly-x86_64
INTERNAL_NETWORK: ["true"]
Base:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules, rules]
- !reference [.nightly_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/base_tests.sh
parallel:
matrix:
- RUNNER:
- aws/fedora-34-x86_64
- aws/fedora-34-aarch64
- aws/fedora-35-x86_64
- aws/fedora-35-aarch64
- aws/rhel-8.5-ga-x86_64
- aws/rhel-8.5-ga-aarch64
- aws/centos-stream-8-x86_64
- aws/centos-stream-8-aarch64
- RUNNER:
- aws/rhel-8.6-nightly-x86_64
- aws/rhel-8.6-nightly-aarch64
- aws/rhel-9.0-nightly-x86_64
- aws/rhel-9.0-nightly-aarch64
- aws/centos-stream-9-x86_64
- aws/centos-stream-9-aarch64
INTERNAL_NETWORK: ["true"]
Manifests:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules, rules]
- !reference [.nightly_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/manifest_tests.sh
parallel:
matrix:
- RUNNER:
- aws/rhel-9.0-nightly-x86_64
INTERNAL_NETWORK: ["true"]
.regression:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules, rules]
- !reference [.nightly_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/${SCRIPT}
parallel:
matrix:
- RUNNER:
- aws/fedora-34-x86_64
- aws/fedora-34-aarch64
- aws/fedora-35-x86_64
- aws/fedora-35-aarch64
- aws/centos-stream-8-x86_64
- aws/centos-stream-8-aarch64
- RUNNER:
- aws/rhel-8.5-ga-x86_64
- aws/rhel-8.5-ga-aarch64
- aws/rhel-8.6-nightly-x86_64
- aws/rhel-8.6-nightly-aarch64
- aws/rhel-9.0-nightly-x86_64
- aws/rhel-9.0-nightly-aarch64
- aws/centos-stream-9-x86_64
- aws/centos-stream-9-aarch64
INTERNAL_NETWORK: ["true"]
regression-bigiso:
extends: .regression
variables:
SCRIPT: regression-bigiso.sh
regression-composer-works-behind-satellite-fallback:
extends: .regression
variables:
SCRIPT: regression-composer-works-behind-satellite-fallback.sh
regression-composer-works-behind-satellite:
extends: .regression
variables:
SCRIPT: regression-composer-works-behind-satellite.sh
regression-excluded-dependency:
extends: .regression
variables:
SCRIPT: regression-excluded-dependency.sh
regression-include-excluded-packages:
extends: .regression
variables:
SCRIPT: regression-include-excluded-packages.sh
regression-old-worker-new-composer:
parallel:
matrix:
- RUNNER:
- aws/rhel-8.5-ga-x86_64
INTERNAL_NETWORK: ["true"]
extends: .regression
variables:
SCRIPT: regression-old-worker-new-composer.sh
regression-no-explicit-rootfs-definition:
extends: .regression
variables:
SCRIPT: regression-no-explicit-rootfs-definition.sh
Image Tests:
stage: test
extends: .terraform
rules:
- !reference [.nightly_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/image_tests.sh
parallel:
matrix:
- RUNNER:
- aws/rhel-8.6-nightly-x86_64
- aws/rhel-8.6-nightly-aarch64
# See https://github.com/osbuild/osbuild-composer/issues/1819 and related issues
- aws/rhel-9.0-nightly-x86_64
- aws/rhel-9.0-nightly-aarch64
INTERNAL_NETWORK: ["true"]
OSTree:
stage: test
extends: .terraform/openstack
rules:
- !reference [.upstream_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/ostree.sh
parallel:
matrix:
- RUNNER:
- rhos-01/fedora-35-x86_64
- rhos-01/rhel-8.5-ga-x86_64
- rhos-01/rhel-8.6-nightly-x86_64-large
- rhos-01/rhel-9.0-nightly-x86_64
- rhos-01/centos-stream-8-x86_64
- rhos-01/centos-stream-9-x86_64
New OSTree:
stage: test
extends: OSTree
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/ostree-ng.sh
parallel:
matrix:
- RUNNER:
- rhos-01/fedora-35-x86_64-large
- rhos-01/rhel-8.6-nightly-x86_64-large
- rhos-01/rhel-9.0-nightly-x86_64-large
- rhos-01/centos-stream-8-x86_64-large
- rhos-01/centos-stream-9-x86_64-large
OSTree simplified installer:
stage: test
extends: OSTree
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/ostree-simplified-installer.sh
parallel:
matrix:
- RUNNER:
- rhos-01/rhel-8.6-nightly-x86_64-large
- rhos-01/centos-stream-8-x86_64-large
- rhos-01/rhel-9.0-nightly-x86_64-large
- rhos-01/centos-stream-9-x86_64-large
OSTree raw image:
stage: test
extends: OSTree
rules:
# run only this edge test on nightly to have some testing for sign-off
# but still reduce duplication with virt-qe Jenkins and increase nightly
# pipelines stability
- !reference [.upstream_rules, rules]
- !reference [.nightly_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/ostree-raw-image.sh
parallel:
matrix:
- RUNNER:
- rhos-01/rhel-8.6-nightly-x86_64-large
- rhos-01/centos-stream-8-x86_64-large
- rhos-01/rhel-9.0-nightly-x86_64-large
- rhos-01/centos-stream-9-x86_64-large
Rebase OSTree:
stage: test
extends: OSTree
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/ostree-rebase.sh
parallel:
matrix:
- RUNNER:
- rhos-01/rhel-8.6-nightly-x86_64-large
- rhos-01/rhel-9.0-nightly-x86_64-large
- rhos-01/centos-stream-8-x86_64-large
- rhos-01/centos-stream-9-x86_64-large
.integration_base:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules, rules]
- !reference [.nightly_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/${SCRIPT}
.rhel_runners: &rhel_runners
RUNNER:
- aws/centos-stream-8-x86_64
- aws/rhel-8.5-ga-x86_64
- aws/rhel-8.6-nightly-x86_64
- aws/rhel-9.0-nightly-x86_64
- aws/centos-stream-9-x86_64
INTERNAL_NETWORK: ["true"]
.integration_rhel:
extends: .integration_base
parallel:
matrix:
- *rhel_runners
.fedora_runners: &fedora_runners
RUNNER:
- aws/fedora-34-x86_64
- aws/fedora-35-x86_64
.integration_fedora:
extends: .integration_base
parallel:
matrix:
- *fedora_runners
.integration:
extends: .integration_base
parallel:
matrix:
- *fedora_runners
- *rhel_runners
koji.sh:
extends: .integration
variables:
SCRIPT: koji.sh
aws.sh:
extends: .integration
variables:
SCRIPT: aws.sh
azure.sh:
extends: .integration
variables:
SCRIPT: azure.sh
# The required GCE image type is not supported on Fedora
gcp.sh:
extends: .integration_rhel
variables:
SCRIPT: gcp.sh
vmware.sh:
extends: .integration
variables:
SCRIPT: vmware.sh
filesystem.sh:
extends: .integration
variables:
SCRIPT: filesystem.sh
cross-distro.sh:
extends: .integration
variables:
SCRIPT: cross-distro.sh
.API_TESTS: &API_TESTS
IMAGE_TYPE:
- aws
- azure
- edge-commit
- gcp
- vsphere
- edge-commit generic.s3
API:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules, rules]
# note: cloud API is not supported for on-prem installations so
# don't run this test case for nightly trees
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/api.sh ${IMAGE_TYPE}
parallel:
matrix:
- <<: *API_TESTS
RUNNER:
- aws/rhel-8.5-ga-x86_64
- aws/rhel-8.6-nightly-x86_64
- aws/rhel-9.0-nightly-x86_64
INTERNAL_NETWORK: ["true"]
- IMAGE_TYPE: ["edge-commit"]
RUNNER:
- aws/fedora-35-x86_64
.libvirt_integration:
stage: test
extends: .terraform/openstack
rules:
- !reference [.upstream_rules, rules]
- !reference [.nightly_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/${SCRIPT}
parallel:
matrix:
- RUNNER:
- rhos-01/centos-stream-8-x86_64
- rhos-01/rhel-8.5-ga-x86_64
- rhos-01/rhel-8.6-nightly-x86_64
- rhos-01/rhel-9.0-nightly-x86_64
- rhos-01/centos-stream-9-x86_64
libvirt.sh:
extends: .libvirt_integration
variables:
SCRIPT: libvirt.sh
generic_s3_http.sh:
extends: .libvirt_integration
variables:
SCRIPT: generic_s3_http.sh
generic_s3_https_secure.sh:
extends: .libvirt_integration
variables:
SCRIPT: generic_s3_https_secure.sh
generic_s3_https_insecure.sh:
extends: .libvirt_integration
variables:
SCRIPT: generic_s3_https_insecure.sh
aws_s3.sh:
extends: .libvirt_integration
variables:
SCRIPT: aws_s3.sh
RHEL 9 on 8:
stage: test
extends: .terraform
rules:
- !reference [.upstream_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/koji.sh
variables:
RUNNER: aws/rhel-8.5-ga-x86_64
INTERNAL_NETWORK: "true"
DISTRO_CODE: rhel-90
Multi-tenancy:
stage: test
extends: .terraform
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule"'
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/multi-tenancy.sh
variables:
RUNNER: aws/rhel-8.5-ga-x86_64
INTERNAL_NETWORK: "true"
Upgrade:
stage: test
extends: .terraform/openstack
rules:
- !reference [.nightly_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/upgrade8to9.sh
variables:
RUNNER: rhos-01/rhel-9.0-nightly-x86_64
artifacts:
paths:
- journal-log.gpg
- ci-artifacts
- install_console.log
- upgrade_console.log
- leapp-preupgrade.log
- leapp-upgrade.log
- leapp-report.txt
- osbuild-upgrade8to9.log
- osbuild-upgrade8to9.json
when: always
NIGHTLY_FAIL:
stage: finish
tags:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true"'
when: on_failure
script:
- schutzbot/slack_notification.sh FAILED ":big-sad:"
NIGHTLY_SUCCESS:
stage: finish
tags:
- shell
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" && $NIGHTLY == "true"'
script:
- schutzbot/slack_notification.sh SUCCESS ":partymeow:"
Installer:
stage: test
extends: .terraform/openstack
rules:
- !reference [.upstream_rules, rules]
- !reference [.nightly_rules, rules]
script:
- schutzbot/deploy.sh
- /usr/libexec/tests/osbuild-composer/installers.sh
parallel:
matrix:
- RUNNER:
- rhos-01/rhel-8.6-nightly-x86_64
- rhos-01/rhel-9.0-nightly-x86_64
- rhos-01/centos-stream-9-x86_64
SCHEDULED_CLOUD_CLEANER:
stage: cleanup
tags:
- terraform
variables:
RUNNER: aws/centos-stream-8-x86_64
INTERNAL_NETWORK: "true"
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" && $CLEANUP == "true"'
script:
- schutzbot/deploy.sh
- schutzbot/scheduled_cloud_cleaner.sh
SonarQube:
stage: test
extends: .terraform
rules:
- if: '$CI_PIPELINE_SOURCE != "schedule" && $CI_COMMIT_BRANCH == "main"'
script:
- schutzbot/sonarqube.sh
variables:
RUNNER: aws/centos-stream-8-x86_64
INTERNAL_NETWORK: "true"
GIT_DEPTH: 0
finish:
stage: finish
dependencies: []
tags:
- shell
script:
- schutzbot/update_github_status.sh finish