From a41ce99521541b32a3fec97da579baa2aef41966 Mon Sep 17 00:00:00 2001 From: Tom Gundersen Date: Thu, 29 Aug 2019 14:24:13 +0200 Subject: [PATCH] test: make the testsuite passive rather than active Let the image be responsible for running its own test, and simply listen for the output from the testsuite. Hook this up with a standard f30 image that contains a simple boot test case, using systemctl to verify that all services started correctly. This replaces the old web-server test, giving similar functionality. The reason for the change is twofold: this way the tests are fully specificed in the pipeline, so easier to reproduce. Moreover, this is less intrusive, as the test does not require network support in the image. Signed-off-by: Tom Gundersen --- test/__main__.py | 21 +++---- test/integration_tests/__init__.py | 4 +- test/integration_tests/run.py | 17 ++---- test/integration_tests/test_case.py | 16 ++--- .../pipelines/f30-boot.json | 30 ++++++++- test/pipelines/web-server.json | 61 ------------------- 6 files changed, 52 insertions(+), 97 deletions(-) rename samples/base-test.json => test/pipelines/f30-boot.json (71%) delete mode 100644 test/pipelines/web-server.json diff --git a/test/__main__.py b/test/__main__.py index 0929e2cf..6bf20172 100644 --- a/test/__main__.py +++ b/test/__main__.py @@ -9,13 +9,8 @@ from test.integration_tests.config import * logging.basicConfig(level=logging.getLevelName(os.environ.get("TESTS_LOGLEVEL", "INFO"))) -def test_web_server_with_curl(): - cmd = ["curl", "-s", "http://127.0.0.1:8888/index"] - logging.info(f"Running curl: {cmd}") - curl = subprocess.run(cmd, capture_output=True) - logging.info(f"Curl returned: code={curl.returncode}, stdout={curl.stdout.decode()}, stderr={curl.stderr.decode()}") - assert curl.returncode == 0 - assert curl.stdout.decode("utf-8").strip() == "hello, world!" +def test_is_system_running(result): + assert result.strip() == "running" def test_timezone(extract_dir): @@ -43,11 +38,11 @@ if __name__ == '__main__': logging.info(f"Using {OUTPUT_DIR} for output images storage.") logging.info(f"Using {OSBUILD} for building images.") - web_server = IntegrationTestCase( - name="web-server", - pipeline="web-server.json", - output_image="web-server.qcow2", - test_cases=[test_web_server_with_curl], + f30_boot = IntegrationTestCase( + name="f30-boot", + pipeline="f30-boot.json", + output_image="f30-boot.qcow2", + test_cases=[test_is_system_running], type=IntegrationTestType.BOOT_WITH_QEMU ) timezone = IntegrationTestCase( @@ -65,7 +60,7 @@ if __name__ == '__main__': type=IntegrationTestType.EXTRACT ) - cases = [web_server, timezone, firewall] + cases = [f30_boot, timezone, firewall] if args.list: print("Available test cases:") diff --git a/test/integration_tests/__init__.py b/test/integration_tests/__init__.py index 09be4705..cf56989b 100644 --- a/test/integration_tests/__init__.py +++ b/test/integration_tests/__init__.py @@ -1,9 +1,9 @@ from .config import * -def evaluate_test(test, name=None): +def evaluate_test(test, arg=None, name=None): try: - test() + test(arg) print(f"{RESET}{BOLD}{name or test.__name__}: Success{RESET}") except AssertionError as e: print(f"{RESET}{BOLD}{name or test.__name__}: {RESET}{RED}Fail{RESET}") diff --git a/test/integration_tests/run.py b/test/integration_tests/run.py index 54af8735..58b54e04 100644 --- a/test/integration_tests/run.py +++ b/test/integration_tests/run.py @@ -6,19 +6,14 @@ import time from .config import * -@contextlib.contextmanager -def boot_image(file_name: str): +def run_image(file_name: str): acceleration = ["-accel", "kvm:hvf:tcg"] - network = ["-net", "nic,model=rtl8139", "-net", "user,hostfwd=tcp::8888-:8888"] - cmd = ["qemu-system-x86_64", "-nographic", "-m", "1024", "-snapshot"] + \ - acceleration + [f"{OUTPUT_DIR}/{file_name}"] + network + silence = ["-nographic", "-monitor", "none", "-serial", "none"] + serial = ["-chardev", "stdio,id=stdio", "-device", "virtio-serial", "-device", "virtserialport,chardev=stdio"] + cmd = ["qemu-system-x86_64", "-m", "1024", "-snapshot"] + \ + acceleration + silence + serial + [f"{OUTPUT_DIR}/{file_name}"] logging.info(f"Booting image: {cmd}") - vm = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - try: - time.sleep(EXPECTED_TIME_TO_BOOT) - yield None - finally: - vm.kill() + return subprocess.run(cmd, capture_output=True, timeout=EXPECTED_TIME_TO_BOOT, encoding="utf-8", check=True) @contextlib.contextmanager diff --git a/test/integration_tests/test_case.py b/test/integration_tests/test_case.py index 650af48b..bc2bb2da 100644 --- a/test/integration_tests/test_case.py +++ b/test/integration_tests/test_case.py @@ -4,7 +4,7 @@ from typing import List, Callable, Any from . import evaluate_test, rel_path from .build import run_osbuild -from .run import boot_image, extract_image +from .run import run_image, extract_image class IntegrationTestType(Enum): @@ -23,16 +23,16 @@ class IntegrationTestCase: def run(self): run_osbuild(rel_path(f"pipelines/{self.pipeline}")) if self.type == IntegrationTestType.BOOT_WITH_QEMU: - self.boot_and_run() + self.run_and_test() else: - self.extract_and_run() + self.extract_and_test() - def boot_and_run(self): - with boot_image(self.output_image): - for test in self.test_cases: - evaluate_test(test) + def run_and_test(self): + r = run_image(self.output_image) + for test in self.test_cases: + evaluate_test(test, r.stdout) - def extract_and_run(self): + def extract_and_test(self): with extract_image(self.output_image) as fstree: for test in self.test_cases: evaluate_test(lambda: test(fstree), name=test.__name__) diff --git a/samples/base-test.json b/test/pipelines/f30-boot.json similarity index 71% rename from samples/base-test.json rename to test/pipelines/f30-boot.json index 920c213e..13067e00 100644 --- a/samples/base-test.json +++ b/test/pipelines/f30-boot.json @@ -1,5 +1,31 @@ { - "name": "base-qcow2", + "name": "f30-boot", + "build": { + "name": "f30-build", + "stages": [ + { + "name": "org.osbuild.dnf", + "options": { + "releasever": "30", + "install_weak_deps": false, + "repos": { + "fedora": { + "name": "Fedora", + "metalink": "https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch", + "gpgkey": "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch" + } + }, + "packages": [ + "dnf", + "e2fsprogs", + "policycoreutils", + "qemu-img", + "systemd" + ] + } + } + ] + }, "stages": [ { "name": "org.osbuild.dnf", @@ -73,7 +99,7 @@ { "name": "org.osbuild.qcow2", "options": { - "filename": "base.qcow2", + "filename": "f30-boot.qcow2", "root_fs_uuid": "76a22bf4-f153-4541-b6c7-0332c0dfaeac" } } diff --git a/test/pipelines/web-server.json b/test/pipelines/web-server.json deleted file mode 100644 index f7d5c9cc..00000000 --- a/test/pipelines/web-server.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "name": "all", - "stages": [ - { - "name": "org.osbuild.dnf", - "options": { - "releasever": "30", - "repos": { - "fedora": { - "name": "Fedora", - "metalink": "https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch", - "gpgkey": "file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch" - } - }, - "packages": [ - "@Core", - "selinux-policy-targeted", - "grub2-pc" - ] - } - }, - { - "name": "org.osbuild.script", - "options": { - "script": "echo root | passwd --stdin root; echo 'SELINUX=disabled' > /etc/selinux/config;" - } - }, - { - "name": "org.osbuild.script", - "options": { - "script": "mkdir -p /var/web; echo 'hello, world!' > /var/web/index; echo -e \"[Unit]\\nDescription=Testing web server\\nAfter=network.target\\n\\n[Service]\\nType=simple\\nExecStart=python3 -m http.server 8888\\nWorkingDirectory=/var/web/\\n\\n[Install]\\nWantedBy=multi-user.target\" > /etc/systemd/system/web-server.service;" - } - }, - { - "name": "org.osbuild.systemd", - "options": { - "enabled_services": [ - "NetworkManager", - "web-server" - ], - "disabled_services": [ - "firewalld" - ] - } - }, - { - "name": "org.osbuild.grub2", - "options": { - "root_fs_uuid": "76a22bf4-f153-4541-b6c7-0332c0dfaeac", - "kernel_opts": "rw" - } - } - ], - "assembler": { - "name": "org.osbuild.qcow2", - "options": { - "filename": "web-server.qcow2", - "root_fs_uuid": "76a22bf4-f153-4541-b6c7-0332c0dfaeac" - } - } -}