⚙ Bring over CI improvements from osbuild-composer

Signed-off-by: Major Hayden <major@redhat.com>
This commit is contained in:
Major Hayden 2020-07-13 11:36:56 -05:00 committed by Christian Kellner
parent e3eccbe491
commit 1a7f286578
8 changed files with 377 additions and 100 deletions

104
schutzbot/Jenkinsfile vendored
View file

@ -11,6 +11,16 @@ pipeline {
}
stages {
stage("Prepare") {
agent { label "schutzbot" }
steps {
sh (
label: "Get environment variables",
script: "env | sort"
)
}
}
stage("Mock") {
// Halt the entire pipeline if a single RPM build fails. That
// could indicate a code problem that needs to be investigated.
@ -18,7 +28,7 @@ pipeline {
parallel {
stage('Fedora 31') {
agent { label "fedora31" }
agent { label "f31cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
@ -32,7 +42,7 @@ pipeline {
}
}
stage('Fedora 32') {
agent { label "fedora32" }
agent { label "f32cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
}
@ -46,9 +56,10 @@ pipeline {
}
}
stage('RHEL 8 CDN') {
agent { label "rhel8" }
agent { label "rhel8cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
sh "schutzbot/ci_details.sh"
@ -59,21 +70,24 @@ pipeline {
)
}
}
// NOTE(mhayden): Disabling this for now since we don't have
// access to these repositories in AWS.
// stage('RHEL 8.3 Nightly') {
// agent { label "rhel83" }
// environment {
// OPENSTACK_CREDS = credentials('psi-openstack-clouds-yaml')
// }
// steps {
// sh "schutzbot/mockbuild.sh"
// stash (
// includes: 'osbuild-mock.repo',
// name: 'rhel83'
// )
// }
// }
stage('RHEL 8.3 Nightly') {
agent { label "rhel83cloudbase && x86_64" }
environment {
AWS_CREDS = credentials('aws-credentials-osbuildci')
NIGHTLY_REPO = credentials('rhel8-nightly-repo')
NIGHTLY_MOCK_TEMPLATE = credentials('rhel8-nightly-mock-template')
}
steps {
sh "schutzbot/ci_details.sh"
retry(3) {
sh "schutzbot/mockbuild.sh"
}
stash (
includes: 'osbuild-mock.repo',
name: 'rhel83'
)
}
}
}
}
stage("Functional Testing") {
@ -82,7 +96,7 @@ pipeline {
parallel {
stage('Fedora 31') {
agent { label "fedora31 && psi" }
agent { label "f31cloudbase && x86_64 && psi" }
environment { TEST_TYPE = "image" }
steps {
unstash 'fedora31'
@ -95,7 +109,7 @@ pipeline {
}
}
stage('Fedora 32') {
agent { label "fedora32 && psi" }
agent { label "f32cloudbase && x86_64 && psi" }
environment { TEST_TYPE = "image" }
steps {
unstash 'fedora32'
@ -108,8 +122,11 @@ pipeline {
}
}
stage('RHEL 8 CDN') {
agent { label "rhel8 && psi" }
environment { TEST_TYPE = "image" }
agent { label "rhel8cloudbase && x86_64 && psi" }
environment {
TEST_TYPE = "image"
RHN_REGISTRATION_SCRIPT = credentials('rhn-register-script-production')
}
steps {
unstash 'rhel8cdn'
run_tests()
@ -120,6 +137,21 @@ pipeline {
}
}
}
stage('RHEL 8.3 Image') {
agent { label "rhel83cloudbase && x86_64" }
environment {
TEST_TYPE = "image"
}
steps {
unstash 'rhel83'
run_tests()
}
post {
always {
preserve_logs('rhel83-image')
}
}
}
}
}
}
@ -130,10 +162,22 @@ pipeline {
void run_tests() {
// Get CI machine details.
sh "schutzbot/ci_details.sh"
sh (
label: "Get CI machine details",
script: "schutzbot/ci_details.sh"
)
// Run the tests from the repository.
sh "schutzbot/run_tests.sh"
// Deploy the Image Builder packages and services.
sh (
label: "Deploy",
script: "schutzbot/deploy.sh"
)
// Run the image tests.
sh (
label: "Image tests",
script: "schutzbot/run_image_tests.sh"
)
}
@ -141,13 +185,19 @@ void run_tests() {
// or failure.
void preserve_logs(test_slug) {
// Save the systemd journal.
sh "sudo journalctl --boot > systemd-journald.log"
// Find any AVCs in the audit log and save those.
sh "sudo grep AVC /var/log/audit/audit.log > selinux-avc.log"
// Make a directory for the log files and move the logs there.
sh "mkdir ${test_slug} && mv *.log ${test_slug}/"
sh "mkdir ${test_slug} && mv *.log *.jpg ${test_slug}/ || true"
// Artifact the logs.
archiveArtifacts (
allowEmptyArchive: true,
artifacts: "${test_slug}/*.log"
artifacts: "${test_slug}/*.log,${test_slug}/*.jpg"
)
}

View file

@ -25,4 +25,18 @@ CI MACHINE SPECS
------------------------------------------------------------------------------
EOF
echo -e "\033[0m"
echo -e "\033[0m"
echo "List of installed packages:"
rpm -qa | sort
echo "------------------------------------------------------------------------------"
# Ensure cloud-init has completely finished on the instance. This ensures that
# the instance is fully ready to go.
while true; do
if [[ -f /var/lib/cloud/instance/boot-finished ]]; then
break
fi
echo -e "\n🤔 Waiting for cloud-init to finish running..."
sleep 5
done

77
schutzbot/deploy.sh Executable file
View file

@ -0,0 +1,77 @@
#!/bin/bash
set -euxo pipefail
function retry {
local count=0
local retries=5
until "$@"; do
exit=$?
count=$(($count + 1))
if [[ $count -lt $retries ]]; then
echo "Retrying command..."
sleep 1
else
echo "Command failed after ${retries} retries. Giving up."
return $exit
fi
done
return 0
}
# Get OS details.
source /etc/os-release
# Register RHEL if we are provided with a registration script.
if [[ -n "${RHN_REGISTRATION_SCRIPT:-}" ]] && ! sudo subscription-manager status; then
sudo chmod +x $RHN_REGISTRATION_SCRIPT
sudo $RHN_REGISTRATION_SCRIPT
fi
# Restart systemd to work around some Fedora issues in cloud images.
sudo systemctl restart systemd-journald
# Remove Fedora's modular repositories to speed up dnf.
sudo rm -f /etc/yum.repos.d/fedora*modular*
# Enable fastestmirror and disable weak dependency installation to speed up
# dnf operations.
echo -e "fastestmirror=1\ninstall_weak_deps=0" | sudo tee -a /etc/dnf/dnf.conf
# Ensure we are using the latest dnf since early revisions of Fedora 31 had
# some dnf repo priority bugs like BZ 1733582.
# NOTE(mhayden): We can exclude kernel updates here to save time with dracut
# and module updates. The system will not be rebooted in CI anyway, so a
# kernel update is not needed.
if [[ $ID == fedora ]]; then
sudo dnf -y upgrade --exclude kernel --exclude kernel-core
fi
# Add osbuild team ssh keys.
cat schutzbot/team_ssh_keys.txt | tee -a ~/.ssh/authorized_keys > /dev/null
# Set up a dnf repository for the RPMs we built via mock.
sudo cp osbuild-mock.repo /etc/yum.repos.d/osbuild-mock.repo
sudo dnf repository-packages osbuild-mock list
# Install the Image Builder packages.
# Note: installing only -tests to catch missing dependencies
retry sudo dnf -y install osbuild-composer-tests
# Set up a directory to hold repository overrides.
sudo mkdir -p /etc/osbuild-composer/repositories
# NOTE(mhayden): RHEL 8.3 is the release we are currently targeting, but the
# release is in beta right now. For RHEL 8.2 (latest release), ensure that
# the production (non-beta content) is used.
if [[ "${ID}${VERSION_ID//./}" == rhel82 ]]; then
sudo cp ${WORKSPACE}/test/external-repos/rhel-8.json \
/etc/osbuild-composer/repositories/rhel-8.json
fi
# Start services.
sudo systemctl enable --now osbuild-rcm.socket
sudo systemctl enable --now osbuild-composer.socket
# Verify that the API is running.
sudo composer-cli status show
sudo composer-cli sources list

View file

@ -1,5 +1,5 @@
#!/bin/bash
set -euxo pipefail
set -euo pipefail
# Colorful output.
function greenprint {
@ -8,11 +8,37 @@ function greenprint {
# Get OS details.
source /etc/os-release
ARCH=$(uname -m)
# Mock is only available in EPEL for RHEL.
if [[ $ID == rhel ]]; then
greenprint "📦 Setting up EPEL repository"
curl -Ls --retry 5 --output /tmp/epel.rpm \
https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
sudo rpm -Uvh /tmp/epel.rpm
fi
# Register RHEL if we are provided with a registration script.
if [[ -n "${RHN_REGISTRATION_SCRIPT:-}" ]] && ! sudo subscription-manager status; then
greenprint "🪙 Registering RHEL instance"
sudo chmod +x $RHN_REGISTRATION_SCRIPT
sudo $RHN_REGISTRATION_SCRIPT
fi
# Install requirements for building RPMs in mock.
greenprint "📦 Installing mock requirements"
sudo dnf -y install createrepo_c make mock python3-pip rpm-build
# Install s3cmd if it is not present.
if ! s3cmd --version; then
if ! s3cmd --version > /dev/null 2>&1; then
greenprint "📦 Installing s3cmd"
sudo pip3 install s3cmd
sudo pip3 -q install s3cmd
fi
# Enable fastestmirror for mock on Fedora.
if [[ $ID == fedora ]]; then
sudo sed -i '/^install_weak_deps=.*/a fastestmirror=1' \
/etc/mock/templates/fedora-branched.tpl
fi
# Jenkins sets a workspace variable as the root of its working directory.
@ -34,33 +60,28 @@ REPO_BUCKET=osbuild-composer-repos
MOCK_REPO_BASE_URL="http://osbuild-composer-repos.s3-website.us-east-2.amazonaws.com"
# Directory to hold the RPMs temporarily before we upload them.
REPO_DIR=repo/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}
REPO_DIR=repo/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}_${ARCH}
# Full URL to the RPM repository after they are uploaded.
REPO_URL=${MOCK_REPO_BASE_URL}/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}
REPO_URL=${MOCK_REPO_BASE_URL}/${JOB_NAME}/${POST_MERGE_SHA}/${ID}${VERSION_ID//./}_${ARCH}
# Print some data.
greenprint "🧬 Using mock config: ${MOCK_CONFIG}"
greenprint "📦 Post merge SHA: ${POST_MERGE_SHA}"
greenprint "📤 RPMS will be uploaded to: ${REPO_URL}"
# Clone osbuild-composer.
# TODO(mhayden): After the next osbuild-composer release, use the latest tag
# in the osbuild-composer repository. We can't do that right now because
# osbuild-composer v12 is missing c0ad652db58059e0e99eb7253b6ba85f25bead3f
# which maks RHEL 8's qemu happy with the image tests.
git clone https://github.com/osbuild/osbuild-composer
# Build source RPMs.
greenprint "🔧 Building source RPMs."
make srpm
git clone --quiet https://github.com/osbuild/osbuild-composer osbuild-composer
make -C osbuild-composer srpm
# Fix RHEL 8 mock template for non-subscribed images.
if [[ $NODE_NAME == *rhel8[23]* ]]; then
greenprint "📋 Updating RHEL 8 mock template for unsubscribed image"
sudo curl --retry 5 -Lsko /etc/mock/templates/rhel-8.tpl \
https://gitlab.cee.redhat.com/snippets/2208/raw
sudo mv $NIGHTLY_MOCK_TEMPLATE /etc/mock/templates/rhel-8.tpl
cat $NIGHTLY_REPO | sudo tee -a /etc/mock/templates/rhel-8.tpl > /dev/null
echo '"""' | sudo tee -a /etc/mock/templates/rhel-8.tpl > /dev/null
fi
# Compile RPMs in a mock chroot
@ -81,7 +102,7 @@ createrepo_c ${REPO_DIR}
# Upload repository to S3.
greenprint "☁ Uploading RPMs to S3"
pushd repo
s3cmd --acl-public sync . s3://${REPO_BUCKET}/
s3cmd --acl-public sync . s3://${REPO_BUCKET}/
popd
# Create a repository file.
@ -94,4 +115,4 @@ enabled=1
gpgcheck=0
# Default dnf repo priority is 99. Lower number means higher priority.
priority=5
EOF
EOF

86
schutzbot/run_image_tests.sh Executable file
View file

@ -0,0 +1,86 @@
#!/bin/bash
set -euo pipefail
# Get OS and architecture details.
source /etc/os-release
ARCH=$(uname -m)
WORKING_DIRECTORY=/usr/libexec/osbuild-composer
IMAGE_TEST_CASE_RUNNER=/usr/libexec/tests/osbuild-composer/osbuild-image-tests
IMAGE_TEST_CASES_PATH=/usr/share/tests/osbuild-composer/cases
PASSED_TESTS=()
FAILED_TESTS=()
# Print out a nice test divider so we know when tests stop and start.
test_divider () {
printf "%0.s-" {1..78} && echo
}
# Get a list of test cases.
get_test_cases () {
TEST_CASE_SELECTOR="${ID}_${VERSION_ID%.*}-${ARCH}*.json"
pushd $IMAGE_TEST_CASES_PATH > /dev/null
ls $TEST_CASE_SELECTOR
popd > /dev/null
}
# Run a test case and store the result as passed or failed.
run_test_case () {
TEST_RUNNER=$1
TEST_CASE_FILENAME=$2
TEST_NAME=$(basename $TEST_CASE_FILENAME)
echo
test_divider
echo "🏃🏻 Running test: ${TEST_NAME}"
test_divider
# Set up the testing command.
TEST_CMD="$TEST_RUNNER -test.v ${IMAGE_TEST_CASES_PATH}/${TEST_CASE_FILENAME}"
# Run the test and add the test name to the list of passed or failed
# tests depending on the result.
if sudo $TEST_CMD 2>&1 | tee ${WORKSPACE}/${TEST_NAME}.log; then
PASSED_TESTS+=("$TEST_NAME")
else
FAILED_TESTS+=("$TEST_NAME")
fi
test_divider
echo
}
# Ensure osbuild-composer-tests is installed.
if ! rpm -qi osbuild-composer-tests > /dev/null 2>&1; then
sudo dnf -y install osbuild-composer-tests
fi
# Change to the working directory.
cd $WORKING_DIRECTORY
# Run each test case.
for TEST_CASE in $(get_test_cases); do
# The fedora_32-x86_64-fedora_iot_commit-boot test has some bugs that
# still need to be worked out. See this bug for details:
# https://github.com/osbuild/osbuild-composer/issues/798
if [[ $TEST_CASE == *fedora_iot_commit* ]]; then
continue
fi
run_test_case $IMAGE_TEST_CASE_RUNNER $TEST_CASE
done
# Print a report of the test results.
test_divider
echo "😃 Passed tests: " "${PASSED_TESTS[@]}"
echo "☹ Failed tests: " "${FAILED_TESTS[@]}"
test_divider
# Exit with a failure if any tests failed.
if [ ${#FAILED_TESTS[@]} -eq 0 ]; then
echo "🎉 All tests passed."
exit 0
else
echo "🔥 One or more tests failed."
exit 1
fi

View file

@ -1,56 +0,0 @@
#!/bin/bash
set -euxo pipefail
# Get OS details.
source /etc/os-release
# Set up a dnf repository for the RPMs we built via mock.
sudo cp osbuild-mock.repo /etc/yum.repos.d/osbuild-mock.repo
dnf repository-packages osbuild-mock list
# Create temporary directories for Ansible.
sudo mkdir -vp /opt/ansible_{local,remote}
sudo chmod -R 777 /opt/ansible_{local,remote}
# Restart systemd to work around some Fedora issues in cloud images.
sudo systemctl restart systemd-journald
# Get the current journald cursor.
export JOURNALD_CURSOR=$(sudo journalctl --quiet -n 1 --show-cursor | tail -n 1 | grep -oP 's\=.*$')
# Add a function to preserve the system journal if something goes wrong.
preserve_journal() {
sudo journalctl --after-cursor=${JOURNALD_CURSOR} > systemd-journald.log
exit 1
}
trap "preserve_journal" ERR
# Write a simple hosts file for Ansible.
echo -e "[test_instances]\nlocalhost ansible_connection=local" > hosts.ini
# Deploy osbuild-composer and osbuild using RPMs built in a mock chroot.
# NOTE(mhayden): Jenkins clones the repository and then merges the code from
# the pull request into the repo. This creates a new SHA that exists only in
# Jenkins. We use ${WORKSPACE} below to tell ansible-osbuild to use the clone
# that Jenkins made for testing osbuild.
export ANSIBLE_CONFIG=ansible-osbuild/ansible.cfg
git clone https://github.com/osbuild/ansible-osbuild.git ansible-osbuild
ansible-playbook \
-i hosts.ini \
-e install_source=os \
ansible-osbuild/playbook.yml
# Ensure the testing package is installed.
sudo dnf -y install osbuild-composer-tests
# Run the image tests from osbuild-composer to stress-test osbuild.
git clone https://github.com/osbuild/osbuild-composer
ansible-playbook \
-e workspace=${WORKSPACE} \
-e journald_cursor="${JOURNALD_CURSOR}" \
-e test_type=${TEST_TYPE:-image} \
-i hosts.ini \
osbuild-composer/schutzbot/test.yml
# Collect the systemd journal anyway if we made it all the way to the end.
sudo journalctl --after-cursor=${JOURNALD_CURSOR} > systemd-journald.log

View file

@ -0,0 +1,19 @@
# SSH keys from members of the osbuild team that are used in CI.
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCxjfFIIlGfCn9iclHymWrEBrTU2lL6tkSGT1ep7dCPzw1jY6WnhQlPXMpePriNxNXlG8cWO9/RFpBd0z9FwNy+kfgh9fuyNY49I+Ma6OyTVBg5hNoFxfRXG5iHtc/SQlnbEFiKpSk4lipo4QZtBtmgAqgkSA6Dzhygb6u5M9ixTIx4WBjuSM0GXQzNjpefyiWu+sIR+h2UrQkKABuuIYQbrjl+FhVmaLvrvyTO2usOtvnYBjhbPwyO72WPjapKd/9hTaqPE1wFy6UF2nXc4Pgw0giQb6sibFTz7NTexW35Q98qpQOWMYKcpgZrlSaHHKZSMhtzO7MdZrOLFUXoS1AeAy4ghtcNrOBTlb5SvP73zz0qBRF2cCO4O0wp5wwqPhvw2ntb3pTLPtdetJ+V50QPnpnXySSnZp2zFwce21bXx67nh9lnhLrZgje7coQnPAFx/cl36ESJygiuPcBw+k18YulYMXUqaBtkwJLkRjDpjTX2e5MJ16oD7sJHc4/W5kyfLvdMsVhdq1CXHGVVOpzogb095VYi0RXFpnZR/1eVgC/R+WVytYfY80rfVOcdAo2GZfnJ5zYRUXJJ9MZkanxx3E7UOikEJN9sUj200z6Cyy0IfIqTbJ1B5f7fd3acRrL4DcYUdFI/1ByNW6F1j7cZiAGOJKNbzXF0T3tf8x0e1Q== major@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDE+WCsyhgXLEBvcNKmEO5w9cYsqlcTUpQLgtHAO8+Ub1bw3UTuiJ/Qd3QmAr6gnb1US0Zs8srTZ5W34jOmYcupmmujLEUUrc/Lj2UzIDWqYi04GD3nzGM/oRWT2glJUXBe63bdy38/GfGdNqp9ZkVgaOEkwVyaxAuNcgNXoEJroSWMStvnDki9SvmXn972QFn+vfQdSt+6z18YFqyE0UhNVX+VE6rezNFRzARCrd+nCc8fVdTRd2vQ+0McButDLzwRXEwLjNOPfXCvoI5EH8+uhOWzLlR9ChfsMcbsipDRHyee0hKWhqg/C8j2DCC9r0zHArPcvLi+qGNIxdSE+KXjWRInr/DSttN/MwDulhJ4uHQQA/PDM1W8yddZpfBv1PaDO+wsm6s2bMN1mzgUPAOZK+i6gaYuoVc89f6+aHJVEPgtqT9Zp+6PD13RWDV6jfi0pfmwRi/CXLlc588oU5D8LEmlUNnUvSNmyV7OJayIK+K6+e7hWGC6/UWess+WSBc= larskarlitski@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQR4bv/n0rVI0ZHV4QoEjNrnHsUFFAcLJ6FWnnJyI31aFXWjjPf3NkbynPqqv3ksk9mj6jJzIBnlo2lZ0kLKIlnblJAyz0GVctxPsBQjzijgLPWTWXS/cLoyLZNS7AsqyTe9rzUATDHmBSje5FaJ6Shas2fybiD5V56fVekgen+sKVBWyFAKsxlWV1EytH5WLn0X0H6K50eCA7sNDfNlGs8k8EXmQPmLOEV55nGI4xBxLmAwx/dn9F3t2EhBwGzw1B6Zc4HA/ayWtJcoARO3gNiazTHKZUz37AAoJ2MnLB698L39aYZ/M55zduSLcyUqF+DBHMfzHH3QRsG0kzv+X9 tgunders@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDV/HNj7nVUp+Yh2hAhHgU+R6X9tLAnApUALhPK8oqH/WG6SjXDiLLEoCSisqu+3+WQ3/UcJeoRl5jpyRAjhxnsz/o5tfLXFDJBZ1/njE9T+e73tjfgS6s+BRpCdEAv6/BUVCxz4B5yI7+5mph0JFpv/soe8uhR6MlxAVxAhaLla4S2UPcn6SciyVznIzrTWgfsSHk/o5hIDVfJ68qYVPWqZWmV86hjE/VhPVgmiqFFh2YsJO/FlH36+wtABcKHbovkCjqQ9PkUqfns+82CNjP/XQ6GJZVK5xpYT59ILjlFwA5s9SYkLcjI+Xy2GGNm4ftNqtoF57q33pfDkxQsYJ+D obudai@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDm5/cfoo+5LBS+0bZ51w8gnB8gUUnUf4Kl4TaLIJF0wBrsywMBXtyRZFhEf04U40bNvWoVjkA3ISvq2Y1PBTRYy78VI5LfOA9/39THVDUZy1l9siSXKpNKx3bUgWGQ17jql7zwzF/Ad4h7E2YD56gtfRXmmF2eFQ2q6OZBXxJgDpbyvBpV0HdmarKTnUSwuvCjkLBSbFSc9MGrFkkkfQjkXtDq6ovzaxr40+u2pEZn/GuwTGFi93Lyovfa/Z4fOrHbJD+AlSmRKUO9Wm+QkSyIuQYeQlr1lN2Mx5tgV2umuuM++mEZq0clfMl7LCz/BfvEUb7w6KxmR++G9nATv1s/ anilsson@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAw6IgsAlMJQlOtJXvtlY1racZPntLiy4+iDwrPMCgbYbsylY5TI2S4JCzC3OsnOF/abozKOhTrX04KOSOPkG8iZjBEUsMX4rQXtdViyec8pAdKOimzN9tdlfC2joW8jPlr/wpKMnMRCQmNDUZIOl1ujyTeY592JE8sj9TTqyc+fk= bcl@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDCk/bA7cpp5ynNr+0SGFFjfdJY1M4EUGqbqbcEQ3uU/CX7AhDEHB2pZMAoCPenoIlB0tz3qTvhQqn9jskiT70qU7bDIRbouXit3Jo0YheYZB7+eD/zxAHropUqjA3Bs6JSoBf2z48FumiFfP+eI0UFJ601Zux2fJCTpLuiboEscegWGyzJrl8/b7XCppNusewM1POyErZDlgHh/2gD7rsupMQKxv3rDOexbGMsA3aGgdrPkvg966tCN5Fz7eBM1RQ3o2ywojTgX9I3U/VeZRJ3w1T1IRDAF/DIMh/dA0afsSXn/7PJUx4NiTRstaBbuYYOMs+PQYSE6o2MLdRXZMQA2Dve/dARkg8LO1gvgNEtNUnDcmVPBrKvULDY9viHrGym9Q0Uo6eaxkqQMjzunV5ozUK6pmuYrMK0dODsVDiB4Ja5CQ6pybAOi+i45FqTT8Wc032KbvydWFpnNCvq31vXfVIysqtGASbhWyvry/Qjr04fkcwQhv6Lnph0pAJr5x9ow5rQrQBJZA23hRXDRuUQqpmnFvzK9TWxb1YQSdyQtpuPplbFUfr5JvyCDwHSBzmPZQUESgddmwCmsO6j0KZVmyJkSbXcWW69LMKGmpb1+Fgg2oo9kAW3e89XWyC7VfLIw5Qog08GtlgcpW92w75nSu3wPMi3ww2g78aG5zgLGw== daherrma@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEApQ0yn8aUPRXoJgGZpWXcOjMXjJgmq9KN/tg0iyK9nuR3uRfdiNGTpgKr7DY9HfdP07R09e/uZ8qnLcXUrXYfJj087JNTMHjT1Ifb0KUB9Mzy82RDUFuefds356arhzhW6YS4cOOgY2GmEt2KhftjdHRic/eElZW9I5mA3Oz/uDk= ckellner@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC8E4VU/3riYCdVAQMBWpwXHiN2/MmUoriYDe6o+hNQ6E1aubdHJ4kYIBxLPYaRvy5sLrbXhsFQH3eNx3HD0TvLiaKU6p9Vj1vEJuTGxB0rIWFATGxvUmBRmUrPNSiWJNCfhjNHTHqboNTf9rX+qgQY5ZqobVpbWHm+sq9QFMtrrF29FiWyowQIFQEPVFx7069Hfk0/JZlEc6rwqX+uiLU8fWP8X1uyXVuNP1xODFX+E/+SC4AbF/wfLlzZ1DVAadzBmCqmqaYQ/dTutXXOeCkiyT8WsPMwp/3FVM+IBwNbyr99fxX0l9j99g7v3vBX3rxV1R4jdQg6htdB7f+ihLAjnAqdNM2ufj0zLn7luJ9icVmV88+DkZf85Y81y/wqqiXEA8BgLlR8zsgnO4RcJoZHq6IErCjvOHBD2rgUAPtJWRLGPUeO53nKyYSobLslIg5dK4klp94DBblV6R+e0tsZQMVvlyj2iBSY/aBR0Cm4em8sXsoZeUQ9+JW6KWXM0yCLAYzhPOsWVi+65qaGjM9JfJKkrdk70Bqx9CixY25WRHiEtUbTpzqNoVil8AB/Ajvofjd36GhQhcng4MKfAUnGm5QQo4k1XT0ExSAwBI3FkkUNZQREhzAdkFdbSo/Iteh2u/OENgJ84nonCUVVFm/vw4nhG+Vmi8E7PJno/jzwoQ== jgiardin@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDQqwlSYW+lD0qlxrol3JQgcN0QSaT9sm3vmhxX18OnP/3XyxZwgnEiJtIDnoZ2FAWOEKTQ9PP0Ab9gfko9mWl6OP5Nj0wT37ZeWYAJoOA3qLw1PCUF8caHs9NcgTEc2Gd+yKIxObQo2xGZQFLBW7owI4S9Rl1a30+5oLFRZJXx8Yd5+b1xFUU/4oOSkc3birHlwTZrFrW6H+AYT4u9ioSvBKtmj7KCqzbRW7p6Sk40p0EoCYU83ZsevzYXTHTAFRHD/gsRS4N7SOUClz+ZnakVBV7fa0ETLkfSmVplhlklBQjBM7OVLeghjlSRXhAcVV63iJfTMNlTjM47MB/C48L+AlH9KMYmbR1ur44NsNGFKTe08B3Q1YfcSR4no6B5iR+zgb6fpDolklAcW5qbHiOOad3Gd8M7VGpDrHZzQe7QGe6fVMKjVq7xfJFgvOaW44F7RTgL6FnT3bNoi6k26enl1O7WlM0v8j/SwtJlSl3I0HpJXLWP/sgr7U590hZSRgG2U5hoXV9YXUCyLFy4pom82CSkAPT2DybdbnhCf4FAnW2CuZMr7KXuVrHC11LA+S2HLN9Fc9f8SA745+0gO5GTvoSWEt6oAgqPWgK6xj3aeHqWN4WQvMHzdftLh4ZxjHd+c7VxreEMQgZJ124W5nUttn4S0xpJHFMADJ44Kjxg6w== jkozol-1@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDMk4+D+GkL/se9RYdQ6mmv0v7cG3gXPiZdLvsbRFk+JMgnLEFUu4uwRN9+huMWdTXYLVcCZVw1l9kmDAOwPKYCGwetOwYI73nYt4n7i9zSQqKknxEO2w+16PvWy6kbgk7zQp9YEqJlIeZbVXiDuFeFyqEH5TxdAfDLu0VG/Db0M1Cor3v7g0/utHWVfS0QiLMQ5xLZ3jAbn/uTjOzsvtA219BTfdx7IVxKmG9/zeLUqyHBcOquI2rUjgHj1FN4yMCob2pSmAfV2u1iQZYHghXlw/kzh4BODjOSWCO9v+621eBdDJ36hdMHWizevxLz9Tj7X1Pwz+4TS/WABXs5MXBFcxx65wpSBr84LfZfWebe+vFe3Kkmdop3FPt7uuZyl8xdRsvfoIG3qZg2izX4IAtXo9tR++yIsEkP/5k/NVwkq158molT0HUruuK9regPRRvf7j6psAH4bMhV3o430eD7ibC5g1yT3kKbGnbJJX+HHL0+lt8IKMPCHWEWOT8uR0x9VUbT4RYFXt53I/ZXHXop+eypEctoKlsgMv1rpPGwSnrhtK1hgf399ph7GDjixZiuzcTP+Ulmj3atB32UHR7mRjarnaCI1w7PJt8pmg9K+8/XUa51q7bap1PGwsJS0lZ3zJEXFPdJfqO8/fkhDb4NYPlX0/bg/LNI9YqPg8CWSw== jkozol-2@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDZ9VgDuObR96fouIDfvzTQxfvMdb4OaK9UPYtQkaVg25a+Yb734Xf0bPFGjM9vtvJmy19qUSlCioqmkEcwheczoDsI/nLBnlRFH3FTmRt3WGJdWcO5MkVXbQAlTqMAlMI86OkeV3jGi5aKYbQ2QJa2ilvQ4JjTEPcAdBmaV9iFgrh9DnIHiAG4RJhrl2BGyf29FIGHusQp7rQSvkwuFPsITjXLcwUKSKbI4+ycwmJN492ovcsiqzG5+YWx12fm3woUWvqFIrt70yc+C7QP68qKJE6qnzWtzZxqtdHR48US0l9xTWB78HGlu0chtTfcF9xT7gokHmx95RWmfXU2fygt1wEYiKNoQnhZTb71ay6fWgzZMOnWrQSyhvKQpIGdPuyevCG7aNtQYTfgCfmPAfZpR1hLQe0feTTLWeS/3aBHUYwYigskFfOZuKy/3Gd8UA2uLeFSOaQ4rYK7sk/FnhGkzxpnW0FR6EXpvWRWdiusjvIZJRyHqb8O3I2uY9aiqV92OaEzwrBlNqqqg0+a7sZOlV8ZtVtzuqBO3yD4tVLGMvO/f28GJ0FnHExrLWHCStqlRiZvqukoJrGBRHJS0j2UK8agwzDM0jHQ4mxbfOwf618puV/YeRhd77GsFcrPD0KONTMeOEoN1jETTBBGZ7Ly/0MsnpP0WR/s8uuvmT7Xaw== msehnout-1@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC1pAobLYTC9BepQi0MViep8sQ0+OpTxP5FSZyiww+IQHpiXPvAzzYL0CYGagvPCI9zPQXx852XjP4T4/Oda5ejcP4+RbT/6ndbUm78xlzRrkYi4dd7OVdmm/1vriK1Q5llcnh+69wao5lXXTZ/j84O2bzAFGnG8MJfMa0gRiqOBCt24EQW1a5NYgTAZQBuD92/pJqc1b1S+hZWerxSdtVrhHi8oRlCwYbVWyhxF+OygfZqe4UyaH5m+eHYLJsxOfwvl9aUvp0ahlOT92l74l0ILuzYSTYIFr/W46F/9tYxiV+rEzjX9niR3sUE9R4U6Jia2aIQXJ6gsftX0Oi5r8nr7DJSPzr4zYMPcxZMEe3nt9ig1dp5DEuT16/e9dgfOuPYBPuEpxVjUPgXeYXIEURRuN9WaCXTkhpAEjIlAzaZsUGMNOVIQsPaHyYnsN/Chk2t4wf/ZIXbGefsTqPLwedbhULPzojHJMfg6Jt2PmUkOhmDtY0jd1vE2cV8/o6refnInf8ePliYZeM7xaF3zZz6ZAmpVAwhkyzJA8AoC4Y4A8+nEWnE641B/l6hXeSd2gdmYSi8FkIstTq6+cUammnJRt/yWLQyYkG4T0RYe9Xp79ZdYC28WOCy6tCzin24a1Zxwf2bWUwnji/01GLxwBHL+wIGHA0noG6XNqbdhRumGQ== msehnout-2@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDCv+bp9I2UqM4msF8D2deAp8s7LjdmH4zHZgioKFRcWMT7ibAVb9B/DNyV9E5VwNs/ofPVjstylHq8R4AJIJGVhVN8woeBjDYdniVJ42Ae7IqoERACDiqASERQSAtYSi+YNZ91jR7ERyHRYspy7lvLhWTyutZbavZa1rk1H9a2pDCfl2RK1Pb+oL41jFzHpupHdBNR0P9feJbbAvwqK6lZI0vMSdY8sJoiVq5MdvB2dy+S+BFs3TNNREaAr4l9XrJ42kmCkBSWVJkmQlCwT8CefZiTKecDNPQE0NI28ABUwQ2T3UElL65Uw5HrWNqIH6CcDfQPDgRfXnfLbPmJMTFQMWRy0H35EdCz0G21M+DfiSxgtOTL2gY872acqtyRwuOdD8/kKD7BuQtlsjpfJtORY+DIMHFF1hfNSWEDztD/4rP9eehYqY3s6UPMTeaowf+6QoAQqDvZG/RH85LhL1FV42N63cIPuTfPErw35hlOhYOUqYrApNtOdsOoEMUunGM= msehnout-3@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDjBCoSJv5HrA0PquLDJYU36oPI0H23AmU2gs8F3k6kBA71N9pHbs/C+YxFYyrsjLqaLwdo44sE2vXB5igJTZN/YpHYytbKHIGdnh3qc0myMvxDLJUxA1kxx3XfT8vbJ0AHrRIJdV6lxfrifI92a0vqRL1pAuWzW/NYytk6DeYDXbUy4xtHOlRIf2WXqZ2pZXsFleaYMdfmUP0y0awe5YDQsNkOBD2HDbU9669jP+UB0nirVuxDhZr1kZmQ8QdQxlePN5LN0RmbNY5Hm1/2SDuXhWh90/fbbWoCkrLTlkAUHckZPmGiEBhD2n4c7jHc25/fhaZsjZITRzHKP5wWW7LE3tXJVMF4eFEzgCoYEr6RpdEwQqQl86WiA0VuLQZbfju5T8t3y6PcGvXD5BF5BJuq710Uqw6y+gy4gNepO4U32jwBz2DDMMBhHsgjSmPfdTvqrjUnlt9NWTXnVDP34Zc5V+RwCgyB09gkwqW+Ae0jDoRSo1hTBSi7v9Gp5R/Hed0= msehnout-4@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDRhju+UtbgUyUzE56Hr8fcRsy0lwk6SKDGn6M7TQ+B0EL5mMTmylbGZ4eP3h+II+GVF5DcoDfXolQvSVn6kSz5lSq/adlbAgGZdxd8Q6N1V+zmMejQf3ZpKQoFWFkmzQoHV+yJkMtWGz//RYqn6eHXvuyDxGyCsdEzBnt10VZL9/g+C45w8l5FlMdGHOhBIkZg1UDLL+2pu8/t4DuPskyr83Thrmb0eZBC4BLLS8Ob9BSZroUgDE1BoSGmfUVnt2Yipjj9qTJ1vWpZPGuqXyCtHHBneJA1b+Hu21CCX/FtSWBNgkbGW2l0rgl8py/GIjqYXPQG0UHkpg6RNlXDARb/GxJVRA2rILX7BsnndKlEh+Kw9Q/IgxpcR5RxZwbH1hYxX1igVyrRW2kAobJhazGV5gBfgm4tZ05dqBL6QX/PqTd6lC5jsyRStwCYIwkh0+Knl2WRagRK/matzvX6MmQXV/vArtY2S4sG73lWN/Af5PnWXQRHRTAyZdSbQC/Ti8c= msehnout-5@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDoVIpowzsQseMML/7I9LcA18G7quBjp2te6GHX9g2IrKh3PxssIS4jQSskRePaCl1yLWhTRDgADkDuOwd8SrQFy7eo7I/GD2wDcdLaWYy4GkdV1M9nhquqpbdLQ2m/3sRhqdHEHadGuw0iYdNgR0M5sNig+xL+G+G4MvjL7hcuaC7GUVbAFWQgePwFWLxjTmX2zmpKQMwk1r4n/i4Wj2GJ7CayxOc9qtnLdnZm9Tj3WA/aRLdbnlM0TCzXrtfO5OLVa9/n3AHY4qqZ1PmRCizLlLtvDeTjhM5qTYCk/xnUNCVnPhe+YqkqXlV6D/D7SI6EIiEd+GI+qJsHt9AgKZpr msehnout-6@redhat.com
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDY/ylCrPBzil4TnZR4tWULpz3QgfBMQyEnMOHDAJNp/FK70hD+PUiRm3UY96pmGXonQvqiDoyPuVh025FkWshPK91Dyq8QD8h25q5C5Cg6kMgBpdGzbX44ksms1KyOHmSZ48MpWw3PFOrlNP1vysr6Imjz9Jixmx4sOZvqKnrbsbOW04gowVzpZM8m048lvf6/KhqeImfeSRc9Rtpos8GqEQVlwRevE1qBON963V1QtFOrm9weoQgb369SdqRRdxaGNAymNh3d78DneOWXmEyBflLSpIDx5I2s/1NB1Dp95Bp3VvlV3CH1HC7LAFKYi+xsz3/KHdgtvgShX6LFSdsp rvykydal@dhcp-lab-144.englab.brq.redhat.com