From f569c38ea994e90888e274502c592e01083d8971 Mon Sep 17 00:00:00 2001 From: Alexander Todorov Date: Fri, 16 Jul 2021 12:48:47 +0300 Subject: [PATCH] Add support for GitLab CI and remove Jenkins related files --- .github/workflows/ci.yml | 18 --- .github/workflows/trigger-gitlab.yml | 57 ++++++++ .gitlab-ci.yml | 69 ++++++++++ schutzbot/Jenkinsfile | 198 --------------------------- schutzbot/deploy.sh | 6 +- schutzbot/mockbuild.sh | 2 +- schutzbot/save_journal.sh | 4 + schutzbot/send_webhook.py | 32 ----- schutzbot/terraform | 1 + schutzbot/update_github_status.sh | 29 ++++ 10 files changed, 164 insertions(+), 252 deletions(-) create mode 100644 .github/workflows/trigger-gitlab.yml create mode 100644 .gitlab-ci.yml delete mode 100644 schutzbot/Jenkinsfile create mode 100755 schutzbot/save_journal.sh delete mode 100755 schutzbot/send_webhook.py create mode 100644 schutzbot/terraform create mode 100755 schutzbot/update_github_status.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ee2d16..6c91326 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,21 +48,3 @@ jobs: - uses: codespell-project/actions-codespell@master with: skip: ./.git,build - - schutzbot: - name: "🍌 Trigger Schutzbot" - runs-on: ubuntu-latest - container: - image: docker.io/library/python:3.7 - steps: - - uses: actions/checkout@v2 - - name: Trigger Schutzbot - env: - EVENT_NAME: ${{ github.event_name }} - WEBHOOK_PAYLOAD: ${{ toJSON(github.event) }} - SQS_REGION: us-east-1 - SQS_QUEUE_URL: "https://sqs.us-east-1.amazonaws.com/933752197999/schutzbot_webhook_sqs-staging" - run: | - #!/bin/bash - pip3 install boto3 botocore - schutzbot/send_webhook.py diff --git a/.github/workflows/trigger-gitlab.yml b/.github/workflows/trigger-gitlab.yml new file mode 100644 index 0000000..efb450d --- /dev/null +++ b/.github/workflows/trigger-gitlab.yml @@ -0,0 +1,57 @@ +# inspired by rhinstaller/anaconda + +name: Trigger GitLab CI +on: [push, pull_request_target] + +jobs: + pr-info: + runs-on: ubuntu-latest + steps: + - name: Query author repository permissions + uses: octokit/request-action@v2.x + id: user_permission + with: + route: GET /repos/${{ github.repository }}/collaborators/${{ github.event.sender.login }}/permission + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # restrict running of tests to users with admin or write permission for the repository + # see https://docs.github.com/en/free-pro-team@latest/rest/reference/repos#get-repository-permissions-for-a-user + # store output if user is allowed in allowed_user job output so it has to be checked in downstream job + - name: Check if user does have correct permissions + if: contains('admin write', fromJson(steps.user_permission.outputs.data).permission) + id: check_user_perm + run: | + echo "User '${{ github.event.sender.login }}' has permission '${{ fromJson(steps.user_permission.outputs.data).permission }}' allowed values: 'admin', 'write'" + echo "::set-output name=allowed_user::true" + + outputs: + allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }} + + trigger-gitlab: + needs: pr-info + if: needs.pr-info.outputs.allowed_user == 'true' + runs-on: ubuntu-latest + env: + SCHUTZBOT_SSH_KEY: ${{ secrets.SCHUTZBOT_SSH_KEY }} + steps: + - name: Clone repository + uses: actions/checkout@v2 + with: + # otherwise we are testing target branch instead of the PR branch (see pull_request_target trigger) + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + + - name: Push to gitlab + run: | + mkdir -p ~/.ssh + echo "${SCHUTZBOT_SSH_KEY}" > ~/.ssh/id_rsa + chmod 400 ~/.ssh/id_rsa + touch ~/.ssh/known_hosts + ssh-keyscan -t rsa gitlab.com >> ~/.ssh/known_hosts + git remote add ci git@gitlab.com:osbuild/ci/koji-osbuild.git + if [ ${{ github.event.pull_request.number }} ]; then + git checkout -b PR-${{ github.event.pull_request.number }} + fi + + git push -f ci diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..44fb27d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,69 @@ +stages: + - init + - rpmbuild + - test + - finish + +.terraform: + tags: + - terraform + artifacts: + paths: + - journal-log + when: always + +init: + stage: init + tags: + - shell + script: + - schutzbot/update_github_status.sh start + +RPM: + stage: rpmbuild + extends: .terraform + script: + - sh "schutzbot/ci_details.sh" + - sh "schutzbot/mockbuild.sh" + after_script: + - schutzbot/update_github_status.sh update + - schutzbot/save_journal.sh + parallel: + matrix: + - RUNNER: + - aws/fedora-33-x86_64 + - aws/fedora-33-aarch64 + - aws/fedora-34-x86_64 + - aws/fedora-34-aarch64 + - aws/rhel-8-x86_64 + - aws/rhel-8-aarch64 + +Testing: + stage: test + extends: .terraform + script: + - sh "schutzbot/ci_details.sh" + - sh "schutzbot/deploy.sh" + - sh "/usr/libexec/tests/koji-osbuild/integration.sh" + after_script: + - schutzbot/update_github_status.sh update + - schutzbot/save_journal.sh + parallel: + matrix: + - RUNNER: + - aws/fedora-33-x86_64 + # https://quay.io/repository/osbuild/postgres available only for x86_64 + # - aws/fedora-33-aarch64 + - aws/fedora-34-x86_64 + # - aws/fedora-34-aarch64 + - RUNNER: + - aws/rhel-8-x86_64 + # - aws/rhel-8-aarch64 + INTERNAL_NETWORK: ["true"] + +finish: + stage: finish + tags: + - shell + script: + - schutzbot/update_github_status.sh finish diff --git a/schutzbot/Jenkinsfile b/schutzbot/Jenkinsfile deleted file mode 100644 index 2522eeb..0000000 --- a/schutzbot/Jenkinsfile +++ /dev/null @@ -1,198 +0,0 @@ -pipeline { - agent none - - environment { - AWS_REGION = "us-east-2" - AWS_BUCKET = "imagebuilder-jenkins-testing-use2" - } - - options { - timestamps() - ansiColor('xterm') - // Cancel the pipeline if it runs for more than three hours. - timeout( - time: 3, - unit: "HOURS" - ) - } - stages { - - stage("Prepare 🤔") { - agent { label "schutzbot" } - options { - // Don't checkout the git repository here. It just clogs - // up the Jenkins disk space and does nothing for us. - skipDefaultCheckout() - } - steps { - sh ( - label: "Get environment variables", - script: "env | sort" - ) - } - } - - stage("Mock build 👷🏻") { - // Halt the pipeline immediately if a single mock build fails. - // A failure to build an RPM is serious and must be - // investigated. - failFast true - - parallel { - stage('F33') { - agent { label "f33cloudbase && x86_64 && aws" } - environment { - AWS_CREDS = credentials('aws-credentials-osbuildci') - } - steps { - sh "schutzbot/ci_details.sh" - retry(3) { - sh "schutzbot/mockbuild.sh" - } - } - } - stage('F34') { - agent { label "f34cloudbase && x86_64 && aws" } - environment { - AWS_CREDS = credentials('aws-credentials-osbuildci') - } - steps { - sh "schutzbot/ci_details.sh" - retry(3) { - sh "schutzbot/mockbuild.sh" - } - } - } - stage('EL8') { - agent { label "rhel8cloudbase && x86_64 && aws" } - environment { - AWS_CREDS = credentials('aws-credentials-osbuildci') - RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production') - } - steps { - sh "schutzbot/ci_details.sh" - retry(3) { - sh "schutzbot/mockbuild.sh" - } - } - } - } - } - - stage("Testing 🍌") { - parallel { - stage('F33 Integration') { - agent { label "f33cloudbase && x86_64 && aws" } - environment { - TEST_TYPE = "integration" - AWS_CREDS = credentials('aws-credentials-osbuildci') - } - steps { - run_tests('integration') - } - post { - always { - preserve_logs('fedora33-integration') - } - } - } - stage('F34 Integration') { - agent { label "f34cloudbase && x86_64 && aws" } - environment { - TEST_TYPE = "integration" - AWS_CREDS = credentials('aws-credentials-osbuildci') - } - steps { - run_tests('integration') - } - post { - always { - preserve_logs('fedora34-integration') - } - } - } - stage('EL8 Integration') { - agent { label "rhel8cloudbase && x86_64 && psi" } - environment { - TEST_TYPE = "integration" - AWS_CREDS = credentials('aws-credentials-osbuildci') - RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production') - } - steps { - run_tests('integration') - } - post { - always { - preserve_logs('rhel8-integration') - } - } - } - } - } - } - - post { - success { - node('schutzbot') { - script { - if (env.BRANCH_NAME == 'main') { - telegramSend "💚 CI passed for koji-osbuild main branch ${env.BUILD_URL}" - } - } - } - } - unsuccessful { - node('schutzbot') { - script { - if (env.BRANCH_NAME == 'main') { - telegramSend "💣 CI failed for koji-osbuild main branch ${env.BUILD_URL}" - } - } - } - } - } - -} - -// Set up a function to hold the steps needed to run the tests so we don't -// need to copy/paste the same lines over and over above. -void run_tests(test_type) { - - // Get CI machine details. - sh ( - label: "Get CI machine details", - script: "schutzbot/ci_details.sh" - ) - - // Deploy the Image Builder packages and services. - sh ( - label: "Deploy", - script: "schutzbot/deploy.sh" - ) - - if (test_type == 'integration') { - sh ( - label: "Integration tests", - script: "/usr/libexec/tests/koji-osbuild/integration.sh" - ) - } - -} - -// Move logs to a unique location and tell Jenkins to capture them on success -// or failure. -void preserve_logs(test_slug) { - - // Save the systemd journal. - sh "journalctl --boot > systemd-journald.log" - - // Make a directory for the log files and move the logs there. - sh "mkdir ${test_slug} && mv *.log *.jpg ${test_slug}/ || true" - - // Artifact the logs. - archiveArtifacts ( - allowEmptyArchive: true, - artifacts: "${test_slug}/*.log,${test_slug}/*.jpg" - ) - -} diff --git a/schutzbot/deploy.sh b/schutzbot/deploy.sh index 81acfe5..63d6dec 100755 --- a/schutzbot/deploy.sh +++ b/schutzbot/deploy.sh @@ -35,7 +35,7 @@ if [[ $ID == rhel ]] && ! rpm -q epel-release; then fi # Register RHEL if we are provided with a registration script. -if [[ -n "${RHN_REGISTRATION_SCRIPT:-}" ]] && ! sudo subscription-manager status; then +if [[ $ID == "rhel" && -n "${RHN_REGISTRATION_SCRIPT:-}" ]] && ! sudo subscription-manager status; then sudo chmod +x $RHN_REGISTRATION_SCRIPT sudo $RHN_REGISTRATION_SCRIPT fi @@ -59,8 +59,8 @@ fi # Set up dnf repositories with the RPMs we want to test sudo tee /etc/yum.repos.d/osbuild.repo << EOF [koji-osbuild] -name=koji-osbuild ${GIT_COMMIT} -baseurl=${DNF_REPO_BASEURL}/koji-osbuild/${DISTRO_VERSION}/${ARCH}/${GIT_COMMIT} +name=koji-osbuild ${CI_COMMIT_SHA} +baseurl=${DNF_REPO_BASEURL}/koji-osbuild/${DISTRO_VERSION}/${ARCH}/${CI_COMMIT_SHA} enabled=1 gpgcheck=0 # Default dnf repo priority is 99. Lower number means higher priority. diff --git a/schutzbot/mockbuild.sh b/schutzbot/mockbuild.sh index 351ef0c..e2269b0 100755 --- a/schutzbot/mockbuild.sh +++ b/schutzbot/mockbuild.sh @@ -11,7 +11,7 @@ source /etc/os-release ARCH=$(uname -m) # Register RHEL if we are provided with a registration script. -if [[ -n "${RHN_REGISTRATION_SCRIPT:-}" ]] && ! sudo subscription-manager status; then +if [[ $ID == "rhel" && -n "${RHN_REGISTRATION_SCRIPT:-}" ]] && ! sudo subscription-manager status; then greenprint "🪙 Registering RHEL instance" sudo chmod +x $RHN_REGISTRATION_SCRIPT sudo $RHN_REGISTRATION_SCRIPT diff --git a/schutzbot/save_journal.sh b/schutzbot/save_journal.sh new file mode 100755 index 0000000..7e708b1 --- /dev/null +++ b/schutzbot/save_journal.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# use tee, otherwise shellcheck complains +sudo journalctl --boot | tee journal-log >/dev/null diff --git a/schutzbot/send_webhook.py b/schutzbot/send_webhook.py deleted file mode 100755 index b4e8c38..0000000 --- a/schutzbot/send_webhook.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python -# Trigger a webhook event for Schutzbot using AWS SQS. -import json -import os - -import boto3 -from botocore import UNSIGNED -from botocore.client import Config - -WEBHOOK_PAYLOAD = os.environ.get("WEBHOOK_PAYLOAD") -EVENT_NAME = os.environ.get("EVENT_NAME") -SQS_QUEUE_URL = os.environ.get("SQS_QUEUE_URL") -SQS_REGION = os.environ.get("SQS_REGION") - -sqs = boto3.client( - 'sqs', - region_name=SQS_REGION, - config=Config( - signature_version=UNSIGNED - ) -) - -payload = json.loads(WEBHOOK_PAYLOAD) -message = { - 'headers': {'X-Github-Event': EVENT_NAME}, - 'payload': payload -} - -response = sqs.send_message( - QueueUrl=SQS_QUEUE_URL, - MessageBody=json.dumps(message) -) diff --git a/schutzbot/terraform b/schutzbot/terraform new file mode 100644 index 0000000..fec2916 --- /dev/null +++ b/schutzbot/terraform @@ -0,0 +1 @@ +3afb50b0f1b4e7476e03151fc8efa81d29ffde49 diff --git a/schutzbot/update_github_status.sh b/schutzbot/update_github_status.sh new file mode 100755 index 0000000..1d4c073 --- /dev/null +++ b/schutzbot/update_github_status.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +if [[ $1 == "start" ]]; then + GITHUB_NEW_STATE="pending" + GITHUB_NEW_DESC="I'm currently testing this commit, be patient." +elif [[ $1 == "finish" ]]; then + GITHUB_NEW_STATE="success" + GITHUB_NEW_DESC="I like this commit!" +elif [[ $1 == "update" ]]; then + if [[ $CI_JOB_STATUS == "canceled" ]]; then + GITHUB_NEW_STATE="failure" + GITHUB_NEW_DESC="Someone told me to cancel this test run." + elif [[ $CI_JOB_STATUS == "failed" ]]; then + GITHUB_NEW_STATE="failure" + GITHUB_NEW_DESC="I'm sorry, something is odd about this commit." + else + exit 0 + fi +else + echo "unknown command" + exit 1 +fi + +curl \ + -u "${SCHUTZBOT_LOGIN}" \ + -X POST \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/osbuild/koji-osbuild/statuses/${CI_COMMIT_SHA}" \ + -d '{"state":"'"${GITHUB_NEW_STATE}"'", "description": "'"${GITHUB_NEW_DESC}"'", "context": "Schutzbot on GitLab", "target_url": "'"${CI_PIPELINE_URL}"'"}'