diff --git a/.travis.yml b/.travis.yml index 7aac0cc3..0dd5c85c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,8 +21,8 @@ jobs: - name: pipeline-noop before_install: sudo apt-get install -y systemd-container script: - - sudo env "PATH=$PATH" python3 -m osbuild --libdir . --output . samples/noop.json - - sudo env "PATH=$PATH" python3 -m osbuild --libdir . --output . samples/noop.json + - sudo env "PATH=$PATH" python3 -m osbuild --libdir . samples/noop.json + - sudo env "PATH=$PATH" python3 -m osbuild --libdir . samples/noop.json - name: f30-boot before_install: sudo apt-get install -y systemd-container yum qemu-kvm script: sudo env "PATH=$PATH" python3 -m test --case f30-boot --build-pipeline samples/build-from-yum.json diff --git a/README.md b/README.md index 4a7a390f..f2df5d28 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ The above pipeline has no base and produces a qcow2 image. ``` usage: python3 -m osbuild [-h] [--build-pipeline PIPELINE] [--store DIRECTORY] - [-l DIRECTORY] -o DIRECTORY + [-l DIRECTORY] PIPELINE Build operating system images @@ -87,11 +87,6 @@ optional arguments: -l DIRECTORY, --libdir DIRECTORY the directory containing stages, assemblers, and the osbuild library - -required named arguments: - -o DIRECTORY, --output DIRECTORY - provide the empty DIRECTORY as output argument to the - last stage ``` ### Running example @@ -99,7 +94,7 @@ required named arguments: You can build basic qcow2 image of Fedora 30 by running a following command: ``` -sudo python3 -m osbuild -o output --libdir . samples/base-qcow2.json +sudo python3 -m osbuild --libdir . samples/base-qcow2.json ``` - Root rights are required because osbuild heavily relies on creating diff --git a/osbuild/__main__.py b/osbuild/__main__.py index 0dd6de18..b929f863 100755 --- a/osbuild/__main__.py +++ b/osbuild/__main__.py @@ -23,9 +23,6 @@ def main(): help="the directory containing stages, assemblers, and the osbuild library") parser.add_argument("--json", action="store_true", help="output results in JSON format") - requiredNamed = parser.add_argument_group('required named arguments') - requiredNamed.add_argument("-o", "--output", dest="output_dir", metavar="DIRECTORY", type=os.path.abspath, - help="provide the empty DIRECTORY as output argument to the last stage", required=True) args = parser.parse_args() with open(args.pipeline_path) as f: @@ -37,7 +34,7 @@ def main(): pipeline.prepend_build_pipeline(build) try: - pipeline.run(args.output_dir, args.store, interactive=not args.json, libdir=args.libdir) + pipeline.run(args.store, interactive=not args.json, libdir=args.libdir) except KeyboardInterrupt: print() print(f"{RESET}{BOLD}{RED}Aborted{RESET}") diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index 9ef0d31e..ee6a4a4a 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -111,7 +111,7 @@ class Assembler: def run(self, tree, build_tree, output_dir=None, interactive=False, check=True, libdir=None): with buildroot.BuildRoot(build_tree) as build_root: if interactive: - print_header(f"Assembling: {self.name}", self.options) + print_header(f"Assembler {self.name}: {self.id}", self.options) args = { "tree": "/run/osbuild/tree", @@ -195,11 +195,11 @@ class Pipeline: finally: subprocess.run(["umount", "--lazy", tmp], check=True) - def run(self, output_dir, store, interactive=False, check=True, libdir=None): + def run(self, store, interactive=False, check=True, libdir=None): os.makedirs("/run/osbuild", exist_ok=True) object_store = objectstore.ObjectStore(store) if self.build: - if not self.build.run(None, store, interactive, check, libdir): + if not self.build.run(store, interactive, check, libdir): return False with self.get_buildtree(object_store) as build_tree: @@ -228,8 +228,9 @@ class Pipeline: libdir=libdir): return False - if self.assembler: - with object_store.get(self.tree_id) as tree: + if self.assembler and not object_store.contains(self.output_id): + with object_store.get(self.tree_id) as tree, \ + object_store.new(self.output_id) as output_dir: if not self.assembler.run(tree, build_tree, output_dir=output_dir, diff --git a/test/__main__.py b/test/__main__.py index 1df003a3..4e14ca62 100644 --- a/test/__main__.py +++ b/test/__main__.py @@ -43,7 +43,6 @@ if __name__ == '__main__': args = parser.parse_args() logging.info(f"Using {OBJECTS} for objects storage.") - logging.info(f"Using {OUTPUT_DIR} for output images storage.") logging.info(f"Using {OSBUILD} for building images.") f30_boot = IntegrationTestCase( diff --git a/test/integration_tests/build.py b/test/integration_tests/build.py index f8802086..38084dc4 100644 --- a/test/integration_tests/build.py +++ b/test/integration_tests/build.py @@ -1,3 +1,4 @@ +import json import logging import subprocess import sys @@ -5,8 +6,8 @@ import sys from .config import * -def run_osbuild(pipeline: str, build_pipeline: str, check=True): - cmd = OSBUILD + ["--store", OBJECTS, "-o", OUTPUT_DIR, pipeline] +def run_osbuild(pipeline: str, build_pipeline: str): + cmd = OSBUILD + ["--json", "--store", OBJECTS, pipeline] if build_pipeline: cmd += ["--build-pipeline", build_pipeline] logging.info(f"Running osbuild: {cmd}") @@ -17,10 +18,10 @@ def run_osbuild(pipeline: str, build_pipeline: str, check=True): print(osbuild.stderr.decode()) print(f"{BOLD}STDOUT{RESET}") print(osbuild.stdout.decode()) - if check: - sys.exit(1) + sys.exit(1) - return osbuild.returncode + result = json.loads(osbuild.stdout.decode()) + return result["tree_id"], result.get("output_id") def build_testing_image(pipeline_full_path, build_pipeline_full_path): diff --git a/test/integration_tests/config.py b/test/integration_tests/config.py index 9f57ee9c..b956e500 100644 --- a/test/integration_tests/config.py +++ b/test/integration_tests/config.py @@ -7,5 +7,4 @@ RESET = "\033[0m" BOLD = "\033[1m" RED = "\033[31m" OBJECTS = os.environ.get("OBJECTS", ".osbuild-test") -OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "output-test") OSBUILD = os.environ.get("OSBUILD", "python3 -m osbuild --libdir .").split(' ') diff --git a/test/integration_tests/run.py b/test/integration_tests/run.py index 7b04e469..0a87a762 100644 --- a/test/integration_tests/run.py +++ b/test/integration_tests/run.py @@ -11,7 +11,7 @@ def run_image(file_name: str): silence = ["-nographic", "-monitor", "none", "-serial", "none"] serial = ["-chardev", "stdio,id=stdio", "-device", "virtio-serial", "-device", "virtserialport,chardev=stdio"] cmd = ["qemu-system-x86_64", "-m", "1024", "-snapshot"] + \ - acceleration + silence + serial + [f"{OUTPUT_DIR}/{file_name}"] + acceleration + silence + serial + [file_name] logging.info(f"Booting image: {cmd}") return subprocess.run(cmd, capture_output=True, timeout=EXPECTED_TIME_TO_BOOT, encoding="utf-8", check=True) @@ -19,7 +19,7 @@ def run_image(file_name: str): @contextlib.contextmanager def extract_image(file_name: str): extract_dir = tempfile.mkdtemp(prefix="osbuild-") - archive = path.join(os.getcwd(), OUTPUT_DIR, file_name) + archive = path.join(os.getcwd(), file_name) subprocess.run(["tar", "xf", archive], cwd=extract_dir, check=True) try: yield extract_dir diff --git a/test/integration_tests/test_case.py b/test/integration_tests/test_case.py index e6f03970..37907c55 100644 --- a/test/integration_tests/test_case.py +++ b/test/integration_tests/test_case.py @@ -5,6 +5,7 @@ from typing import List, Callable, Any from . import evaluate_test, rel_path from .build import run_osbuild from .run import run_image, extract_image +from test.integration_tests.config import * class IntegrationTestType(Enum): @@ -22,18 +23,18 @@ class IntegrationTestCase: type: IntegrationTestType def run(self): - run_osbuild(rel_path(f"pipelines/{self.pipeline}"), self.build_pipeline) + tree_id, output_id = run_osbuild(rel_path(f"pipelines/{self.pipeline}"), self.build_pipeline) if self.type == IntegrationTestType.BOOT_WITH_QEMU: - self.run_and_test() + self.run_and_test(output_id) else: - self.extract_and_test() + self.extract_and_test(output_id) - def run_and_test(self): - r = run_image(self.output_image) + def run_and_test(self, output_id): + r = run_image(f"{OBJECTS}/refs/{output_id}/{self.output_image}") for test in self.test_cases: evaluate_test(test, r.stdout) - def extract_and_test(self): - with extract_image(self.output_image) as fstree: + def extract_and_test(self, output_id): + with extract_image(f"{OBJECTS}/refs/{output_id}/{self.output_image}") as fstree: for test in self.test_cases: evaluate_test(lambda: test(fstree), name=test.__name__)