Tests: add new manifest tests using osbuild/images cache
Add new implementation of the manifest tests, which goal is to ensure that the osbuild behavior didn't change. This is ensured by comparing image-info report produced for image artifact built using older (known-to-be-good) osbuild version and the latest osbuild version (potentially from a PR). Previously, we used the osbuild/manifest-db repository, which contained pre-generated manifests with their corresponding image-info report. Unfortunately, this setup prooved to be cumbersome to maintain and keep updated. We are already building images for known manifests in the osbuild/images repository. These are then uploaded to AWS S3 cache. The images are built with a pinned osbuild version, which will be always older than the one that we would be using for image build in osbuild PR. So the intention of this new script is to take advantage of the osbuild/images S3 cache. As part of the test case (for a specific distro / arch / image_type / config): - download the manifest from S3 - download the image artifact, built from the manifest, from S3 - generate image-info report for the downloaded image - rebuild the downloaded manifest using current version of osbuild - generate image-info report for the rebuilt image - compare the two image-info reports. If there is no difference, the test case PASS, otherwise it will FAIL. Signed-off-by: Tomáš Hozza <thozza@redhat.com>
This commit is contained in:
Tomáš Hozza
Tomáš Hozza
parent
035781ea1c
commit
a244003e6e
1 changed files with 465 additions and 0 deletions
465
test/cases/manifest_tests
Executable file
465
test/cases/manifest_tests
Executable file
|
|
@ -0,0 +1,465 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
"""
|
||||
Test manifest building consistency using osbuild-image-info
|
||||
|
||||
The test case downloads the image build cache (manifests, build info and artifacts) for the osbuild/images repository
|
||||
from S3. It then runs osbuild-image-info on the downloaded image artifact file and produces an image-info report.
|
||||
The test case then builds the manifest using the current version of osbuild and produces the image artifact.
|
||||
It runs osbuild-image-info on the newly built image artifact file and produces an image-info report.
|
||||
Finally, it compares the image-info reports and fails if they are different.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
OSBUILD_IMAGES_REPO_URL = os.environ.get("OSBUILD_IMAGES_REPO_URL", "https://github.com/osbuild/images.git")
|
||||
|
||||
OS_RELEASE_FILE = "/etc/os-release"
|
||||
|
||||
|
||||
def read_osrelease() -> Dict[str, str]:
|
||||
"""Read Operating System Information from `os-release`
|
||||
|
||||
This creates a dictionary with information describing the running operating system. It reads the information from
|
||||
the path array provided as `paths`. The first available file takes precedence. It must be formatted according to
|
||||
the rules in `os-release(5)`.
|
||||
"""
|
||||
osrelease = {}
|
||||
|
||||
with open(OS_RELEASE_FILE, encoding="utf-8") as orf:
|
||||
for line in orf:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if line[0] == "#":
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
osrelease[key] = value.strip('"')
|
||||
|
||||
return osrelease
|
||||
|
||||
|
||||
def get_host_distro() -> str:
|
||||
"""
|
||||
Get the host distro version based on data in the os-release file.
|
||||
The format is <distro>-<version> (e.g. fedora-41).
|
||||
"""
|
||||
osrelease = read_osrelease()
|
||||
return f"{osrelease['ID']}-{osrelease['VERSION_ID']}"
|
||||
|
||||
|
||||
def get_host_arch() -> str:
|
||||
"""
|
||||
Get the host architecture.
|
||||
"""
|
||||
return os.uname().machine
|
||||
|
||||
|
||||
def manifest_pipeline_names(path: str) -> List[str]:
|
||||
"""
|
||||
Read the manifest file and return a list of pipeline names
|
||||
"""
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
manifest = json.load(f)
|
||||
|
||||
return [pipeline["name"] for pipeline in manifest["pipelines"]]
|
||||
|
||||
|
||||
def find_manifest_file(build_path: str) -> str:
|
||||
"""
|
||||
Return the path to the manifest file in the build directory
|
||||
"""
|
||||
return os.path.join(build_path, "manifest.json")
|
||||
|
||||
|
||||
def find_image_file(build_path: str, export_name: Optional[str] = None) -> str:
|
||||
"""
|
||||
Find the path to the image by searching for the file under the directory named 'export_name'. If the name of the
|
||||
export pipeline is not provided, determine it by reading the manifest to get the name of the last pipeline.
|
||||
Raises RuntimeError if no or multiple files are found in the expected path.
|
||||
"""
|
||||
if export_name is None:
|
||||
manifest_file = find_manifest_file(build_path)
|
||||
export_name = manifest_pipeline_names(manifest_file)[-1]
|
||||
files = os.listdir(os.path.join(build_path, export_name))
|
||||
if len(files) > 1:
|
||||
error = "Multiple files found in build path while searching for image file"
|
||||
error += "\n".join(files)
|
||||
raise RuntimeError(error)
|
||||
|
||||
if len(files) == 0:
|
||||
raise RuntimeError("No found in build path while searching for image file")
|
||||
|
||||
return os.path.join(build_path, export_name, files[0])
|
||||
|
||||
|
||||
def checkout_images_repo(ref, workdir: os.PathLike) -> str:
|
||||
"""
|
||||
Checkout the 'images' repository at a specific commit and return the path to the directory
|
||||
If the repository is already checked-out, switch to the specified commit.
|
||||
"""
|
||||
images_path = os.path.join(workdir, "images")
|
||||
|
||||
if not os.path.exists(images_path):
|
||||
print(f"Checking out '{OSBUILD_IMAGES_REPO_URL}' repository at ref '{ref}'")
|
||||
try:
|
||||
subprocess.check_call(
|
||||
["git", "clone", "--depth=1", "--no-single-branch", OSBUILD_IMAGES_REPO_URL, "images"],
|
||||
cwd=workdir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Failed to clone 'images' repository: {e.stdout.decode()}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"'images' repository is already checked-out at '{images_path}'")
|
||||
|
||||
subprocess.check_call(["git", "checkout", ref], cwd=images_path, stdout=subprocess.DEVNULL)
|
||||
return images_path
|
||||
|
||||
|
||||
def download_image_build_artifact(images_path: os.PathLike, build_dir: os.PathLike) -> None:
|
||||
"""
|
||||
Download the image build artifact from S3 for a specific image build.
|
||||
"""
|
||||
cmd = ["./test/scripts/dl-one-image-build-cache", build_dir]
|
||||
print(" ".join(cmd))
|
||||
try:
|
||||
subprocess.check_call(cmd, cwd=images_path, env=os.environ)
|
||||
except subprocess.CalledProcessError as _:
|
||||
print("⚠️ Failed to download image build cache")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def download_image_build_cache_md(
|
||||
images_path, output_dir, distros: List[str], arch: str, configs: Optional[List[str]] = None,
|
||||
image_types: Optional[List[str]] = None, skip_img_types: Optional[List[str]] = None) -> None:
|
||||
"""
|
||||
Download the image build cache metadata from S3 for a specific distro / arch / configs.
|
||||
The image artifacts is not downloaded, only the metadata.
|
||||
"""
|
||||
cmd = [
|
||||
"./test/scripts/dl-image-build-cache",
|
||||
"--arch", arch,
|
||||
"--output", output_dir,
|
||||
]
|
||||
for distro in distros:
|
||||
cmd += ["--distro", distro]
|
||||
for config in configs or []:
|
||||
cmd += ["--config", config]
|
||||
|
||||
# The image_types and skip_img_types are mutually exclusive,
|
||||
# but this is enforced in the argument parser.
|
||||
for imag_type in image_types or []:
|
||||
cmd += ["--image-type", imag_type]
|
||||
for skip_img_type in skip_img_types or []:
|
||||
cmd += ["--skip-image-type", skip_img_type]
|
||||
|
||||
print(" ".join(cmd))
|
||||
try:
|
||||
subprocess.check_call(cmd, cwd=images_path, env=os.environ)
|
||||
except subprocess.CalledProcessError as _:
|
||||
print("⚠️ Failed to download image build cache")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def gen_image_info_report(image_path: str, report_path: str) -> None:
|
||||
"""
|
||||
Run osbuild-image-info on the image file and return the parsed JSON output
|
||||
"""
|
||||
cmd = ["sudo", "osbuild-image-info", image_path]
|
||||
output = subprocess.check_output(cmd)
|
||||
with open(report_path, "w", encoding="utf-8") as f:
|
||||
f.write(output.decode())
|
||||
|
||||
|
||||
def diff_files(file1: str, file2: str) -> str:
|
||||
"""
|
||||
Run diff on two files and return the output
|
||||
"""
|
||||
cmd = [
|
||||
"diff", "-u",
|
||||
"-I", r'"[0-9a-f]\{8\}-\([0-9a-f]\{4\}-\)\{3\}[0-9a-f]\{12\}"', # Ignore UUIDs
|
||||
"-I", r'"[0-9a-zA-Z]\{6\}-\([0-9a-zA-Z]\{4\}-\)\{5\}[0-9a-zA-Z]\{6\}"', # Ignore LVM quasi-UUIDs
|
||||
"-I", r'"volid": "[0-9a-f]\{8\}"', # Ignore volid
|
||||
file1, file2,
|
||||
]
|
||||
print(" ".join(cmd))
|
||||
run = subprocess.run(cmd, stdout=subprocess.PIPE, check=False)
|
||||
return run.stdout.decode()
|
||||
|
||||
|
||||
class OSBuild:
|
||||
def __init__(self, store, outdir):
|
||||
self.store = store
|
||||
self.outdir = outdir
|
||||
|
||||
def run(self, manifest, exports, checkpoints=None) -> Tuple[int, str, str]:
|
||||
cmd = [
|
||||
"sudo",
|
||||
"osbuild",
|
||||
"--cache-max-size", "unlimited",
|
||||
"--store", os.fspath(self.store),
|
||||
"--output-directory", os.fspath(self.outdir),
|
||||
os.fspath(manifest)
|
||||
]
|
||||
|
||||
for checkpoint in checkpoints or []:
|
||||
cmd += [
|
||||
"--checkpoint", checkpoint
|
||||
]
|
||||
|
||||
for export in exports:
|
||||
cmd += [
|
||||
"--export", export
|
||||
]
|
||||
|
||||
print(" ".join(cmd))
|
||||
run = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False)
|
||||
|
||||
return run.returncode, run.stdout.decode(), run.stderr.decode()
|
||||
|
||||
|
||||
def run_manifest_behavior_test(
|
||||
build_dir: str, results_dir: str, osbuild_store: str, rm_artifacts_after_test: bool = False) -> None:
|
||||
"""
|
||||
This function implements the test case for a single image build cache directory which contains:
|
||||
- the manifest.json file
|
||||
- the image artifact file built by an older version of osbuild pinned in the images repository
|
||||
|
||||
The test case does the following:
|
||||
- runs osbuild-image-info on the downloaded image artifact file and produces image-info report
|
||||
- builds the manifest using the current version of osbuild and produces image artifact
|
||||
- runs osbuild-image-info on the newly built image artifact file and produces image-info report
|
||||
- compares the image-info reports
|
||||
|
||||
The test case fails if the image-info reports are different
|
||||
|
||||
The function can optionally remove the downloaded and rebuilt image artifacts after the test case
|
||||
to save disk space.
|
||||
"""
|
||||
manifest = find_manifest_file(build_dir)
|
||||
manifest_pipelines = manifest_pipeline_names(manifest)
|
||||
downloaded_image = find_image_file(build_dir)
|
||||
downloaded_image_iminfo = os.path.join(results_dir, "downloaded_image_iminfo.json")
|
||||
print(f"📜 Generating image info report for downloaded image '{downloaded_image}' to '{downloaded_image_iminfo}'")
|
||||
gen_image_info_report(downloaded_image, downloaded_image_iminfo)
|
||||
|
||||
if rm_artifacts_after_test:
|
||||
print("🗑️ Removing downloaded image artifact")
|
||||
os.remove(downloaded_image)
|
||||
|
||||
rebuild_dir = os.path.join(build_dir, "rebuild")
|
||||
os.makedirs(rebuild_dir, exist_ok=True)
|
||||
osbuild = OSBuild(osbuild_store, rebuild_dir)
|
||||
print("Rebuilding the image artifact using installed osbuild version")
|
||||
retcode, stdout, stderr = osbuild.run(manifest, [manifest_pipelines[-1]], ["build"])
|
||||
with open(os.path.join(rebuild_dir, "image_rebuild_osbuild.log"), "w", encoding="utf-8") as f:
|
||||
f.write(stdout)
|
||||
f.write(stderr)
|
||||
if retcode != 0:
|
||||
raise RuntimeError(f"Failed to rebuild the image artifact:\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}")
|
||||
|
||||
rebuilt_image = find_image_file(rebuild_dir, manifest_pipelines[-1])
|
||||
rebuilt_image_iminfo = os.path.join(results_dir, "rebuilt_image_iminfo.json")
|
||||
print(f"📜 Generating image info report for rebuilt image: '{rebuilt_image}' to '{rebuilt_image_iminfo}'")
|
||||
gen_image_info_report(rebuilt_image, rebuilt_image_iminfo)
|
||||
|
||||
if rm_artifacts_after_test:
|
||||
print("🗑️ Removing the dir with rebuilt image artifact")
|
||||
# NB: use sudo to remove the dir, because the files are owned by root after osbuild run
|
||||
subprocess.run(["sudo", "rm", "-rf", rebuild_dir], check=False)
|
||||
|
||||
diff = diff_files(downloaded_image_iminfo, rebuilt_image_iminfo)
|
||||
with open(os.path.join(results_dir, "iminfo.diff"), "w", encoding="utf-8") as f:
|
||||
f.write(diff)
|
||||
if diff:
|
||||
raise RuntimeError(f"Image info reports are different:\n{diff}")
|
||||
|
||||
|
||||
def get_argparser():
|
||||
|
||||
class ExtendAction(argparse.Action):
|
||||
"""
|
||||
Custom argparse action to append multiple values to a list option
|
||||
to prevent overwriting the list with each new value.
|
||||
|
||||
This may be removed when Python 3.8 is the minimum supported version.
|
||||
"""
|
||||
|
||||
def __call__(self, parser, namespace, values, option_string=None):
|
||||
items = getattr(namespace, self.dest) or []
|
||||
items.extend(values)
|
||||
setattr(namespace, self.dest, items)
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.register('action', 'extend', ExtendAction)
|
||||
|
||||
parser.add_argument(
|
||||
"--distro",
|
||||
metavar="DISTRO",
|
||||
action="extend",
|
||||
nargs="+",
|
||||
help="Distro to test. Default is the host distro."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--image-type", action="extend", metavar="TYPE", nargs="+",
|
||||
help="Image type for which the image build cache is downloaded. Can be specified multiple times. " +
|
||||
"If not provided, all image types are downloaded. " +
|
||||
"The option is mutually exclusive with --skip-image-type.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-image-type", action="extend", metavar="TYPE_GLOB", nargs="+",
|
||||
help="Image types to skip when downloading the image build cache. Can be specified multiple times. " +
|
||||
"This is useful for image types which can't be analyzed using osbuild-image-info.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--config", action="extend", metavar="NAME_GLOB", nargs="+",
|
||||
help="Config name globs used to filter which image build cache files are downloaded. " +
|
||||
"Can be specified multiple times. If not provided, all configs are downloaded.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--images-ref",
|
||||
metavar="REF",
|
||||
default="main",
|
||||
help="Git ref to checkout in the osbuild/images repository."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--osb-store",
|
||||
metavar="PATH",
|
||||
type=os.path.abspath,
|
||||
default=None,
|
||||
help="Directory where intermediary os trees are stored."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workdir",
|
||||
metavar="PATH",
|
||||
type=os.path.abspath,
|
||||
default="./osbuild-manifest-tests-workdir",
|
||||
help="Working directory where the images repository is checked out and the image build cache is downloaded."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--results-dir",
|
||||
metavar="PATH",
|
||||
type=os.path.abspath,
|
||||
default="./osbuild-manifest-tests-results",
|
||||
help="Directory where the test results are stored."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chunk-size", nargs=2, metavar=("CHUNK_NUMBER", "TOTAL_CHUNKS"),
|
||||
type=int, default=None,
|
||||
help="Run tests only for a specific chunk of the image build cache. " +
|
||||
"The first argument is the chunk number (starting from 1) and the second argument is the total " +
|
||||
"number of chunks. This is useful for running the tests in parallel."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--rm-artifacts-after-test",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Remove the downloaded and rebuilt image artifacts after the test case. " +
|
||||
"This is useful for saving disk space."
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
parser = get_argparser()
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.distro:
|
||||
args.distro = [get_host_distro()]
|
||||
|
||||
chunk_number, total_chunks = None, None
|
||||
if args.chunk_size:
|
||||
chunk_number, total_chunks = args.chunk_size
|
||||
if chunk_number < 1 or total_chunks < 1:
|
||||
parser.error("Both values for '--chunk-size' must be greater than zero.")
|
||||
if chunk_number > total_chunks:
|
||||
parser.error("The chunk number must be less than or equal to the total number of chunks.")
|
||||
|
||||
if args.image_type and args.skip_image_type:
|
||||
parser.error("Options --image-type and --skip-image-type are mutually exclusive.")
|
||||
|
||||
workdir = args.workdir
|
||||
os.makedirs(workdir, exist_ok=True)
|
||||
print(f"👷 Using working directory: {workdir}")
|
||||
os.chdir(workdir)
|
||||
|
||||
osbuild_store_dir = args.osb_store or os.path.join(workdir, "osbuild-store")
|
||||
os.makedirs(osbuild_store_dir, exist_ok=True)
|
||||
print(f"💾 Using osbuild store directory: {osbuild_store_dir}")
|
||||
|
||||
# Checkout the images repository, since we will need scripts from it to download the image build cache
|
||||
images_path = checkout_images_repo(args.images_ref, workdir)
|
||||
|
||||
# Create the directory where the image build cache will be downloaded
|
||||
image_build_cache = os.path.join(workdir, "image-build-cache")
|
||||
os.makedirs(image_build_cache, exist_ok=True)
|
||||
|
||||
download_image_build_cache_md(
|
||||
images_path, image_build_cache, args.distro, get_host_arch(),
|
||||
args.config, args.image_type, args.skip_image_type
|
||||
)
|
||||
|
||||
# The test case is run for every directory in the image build cache directory
|
||||
test_cases = sorted(os.listdir(image_build_cache))
|
||||
|
||||
if not test_cases:
|
||||
print("⚠️ No image build cache directories found -> nothing to test", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"📦 Found {len(test_cases)} image build cache directories")
|
||||
if chunk_number and total_chunks:
|
||||
all_test_cases = test_cases
|
||||
print(f"📦 Will run subset of tests for chunk {chunk_number} of {total_chunks}:")
|
||||
chunk_size = len(test_cases) // total_chunks
|
||||
chunk_size_remainder = len(test_cases) % total_chunks
|
||||
|
||||
# determine the chunk index range
|
||||
start = 0
|
||||
end = 0
|
||||
for i in range(chunk_number):
|
||||
current_chunk_size = chunk_size + 1 if i < chunk_size_remainder else chunk_size
|
||||
start = end
|
||||
end = start + current_chunk_size
|
||||
|
||||
test_cases = test_cases[start:end]
|
||||
idx_range = range(start, end)
|
||||
for i, test_case in enumerate(all_test_cases):
|
||||
print(f" {'🟢' if i in idx_range else '🚫'} {test_case}")
|
||||
|
||||
# Dictionary holding the test case name as key and a boolean indicating if the test case failed as value
|
||||
test_cases_failed: Dict[str, bool] = {}
|
||||
print(f"🏃 Running {len(test_cases)} test cases:\n{os.linesep.join(test_cases)}")
|
||||
for test_case in test_cases:
|
||||
print(f"🏃 Running test case for {test_case}")
|
||||
test_case_build_dir = os.path.join(image_build_cache, test_case)
|
||||
download_image_build_artifact(images_path, test_case_build_dir)
|
||||
test_case_results_dir = os.path.join(args.results_dir, test_case)
|
||||
os.makedirs(test_case_results_dir, exist_ok=True)
|
||||
try:
|
||||
run_manifest_behavior_test(
|
||||
test_case_build_dir, test_case_results_dir, osbuild_store_dir, args.rm_artifacts_after_test)
|
||||
# pylint: disable=broad-exception-caught
|
||||
except Exception as e:
|
||||
print(f"❌ {test_case} FAILED")
|
||||
print(e)
|
||||
test_cases_failed[test_case] = True
|
||||
else:
|
||||
print(f"✅ {test_case} PASSED")
|
||||
test_cases_failed[test_case] = False
|
||||
|
||||
print("Test results:")
|
||||
for test_case, failed in test_cases_failed.items():
|
||||
print(f" {'❌' if failed else '✅'} {test_case}")
|
||||
|
||||
if any(test_cases_failed.values()):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue