osbuild: store outputs in objectstore

Treat outputs like we treat trees: store them in the object store. This
simplifies using osbuild and allows returning a cached version if one is
available.

This makes the `--output` parameter redundant. Remove it.

.travis.yml

@@ -21,8 +21,8 @@ jobs:
     - name: pipeline-noop
       before_install: sudo apt-get install -y systemd-container
       script:
-        - sudo env "PATH=$PATH" python3 -m osbuild --libdir . --output . samples/noop.json
+        - sudo env "PATH=$PATH" python3 -m osbuild --libdir . samples/noop.json
-        - sudo env "PATH=$PATH" python3 -m osbuild --libdir . --output . samples/noop.json
+        - sudo env "PATH=$PATH" python3 -m osbuild --libdir . samples/noop.json
     - name: f30-boot
       before_install: sudo apt-get install -y systemd-container yum qemu-kvm
       script: sudo env "PATH=$PATH" python3 -m test --case f30-boot --build-pipeline samples/build-from-yum.json

README.md

@@ -70,7 +70,7 @@ The above pipeline has no base and produces a qcow2 image.
 
 ```
 usage: python3 -m osbuild [-h] [--build-pipeline PIPELINE] [--store DIRECTORY]
-                   [-l DIRECTORY] -o DIRECTORY
+                   [-l DIRECTORY]
                    PIPELINE
 
 Build operating system images
@@ -87,11 +87,6 @@ optional arguments:
   -l DIRECTORY, --libdir DIRECTORY
                         the directory containing stages, assemblers, and the
                         osbuild library
-
-required named arguments:
-  -o DIRECTORY, --output DIRECTORY
-                        provide the empty DIRECTORY as output argument to the
-                        last stage
 ```
 
 ### Running example
@@ -99,7 +94,7 @@ required named arguments:
 You can build basic qcow2 image of Fedora 30 by running a following command:
 
 ```
-sudo python3 -m osbuild -o output --libdir . samples/base-qcow2.json
+sudo python3 -m osbuild --libdir . samples/base-qcow2.json
 ```
 
 - Root rights are required because osbuild heavily relies on creating

osbuild/__main__.py

@@ -23,9 +23,6 @@ def main():
                         help="the directory containing stages, assemblers, and the osbuild library")
     parser.add_argument("--json", action="store_true",
                         help="output results in JSON format")
-    requiredNamed = parser.add_argument_group('required named arguments')
-    requiredNamed.add_argument("-o", "--output", dest="output_dir", metavar="DIRECTORY", type=os.path.abspath,
-                               help="provide the empty DIRECTORY as output argument to the last stage", required=True)
     args = parser.parse_args()
 
     with open(args.pipeline_path) as f:
@@ -37,7 +34,7 @@ def main():
         pipeline.prepend_build_pipeline(build)
 
     try:
-        pipeline.run(args.output_dir, args.store, interactive=not args.json, libdir=args.libdir)
+        pipeline.run(args.store, interactive=not args.json, libdir=args.libdir)
     except KeyboardInterrupt:
         print()
         print(f"{RESET}{BOLD}{RED}Aborted{RESET}")

osbuild/pipeline.py

@@ -111,7 +111,7 @@ class Assembler:
     def run(self, tree, build_tree, output_dir=None, interactive=False, check=True, libdir=None):
         with buildroot.BuildRoot(build_tree) as build_root:
             if interactive:
-                print_header(f"Assembling: {self.name}", self.options)
+                print_header(f"Assembler {self.name}: {self.id}", self.options)
 
             args = {
                 "tree": "/run/osbuild/tree",
@@ -195,11 +195,11 @@ class Pipeline:
                 finally:
                     subprocess.run(["umount", "--lazy", tmp], check=True)
 
-    def run(self, output_dir, store, interactive=False, check=True, libdir=None):
+    def run(self, store, interactive=False, check=True, libdir=None):
         os.makedirs("/run/osbuild", exist_ok=True)
         object_store = objectstore.ObjectStore(store)
         if self.build:
-            if not self.build.run(None, store, interactive, check, libdir):
+            if not self.build.run(store, interactive, check, libdir):
                 return False
 
         with self.get_buildtree(object_store) as build_tree:
@@ -228,8 +228,9 @@ class Pipeline:
                                              libdir=libdir):
                                 return False
 
-            if self.assembler:
+            if self.assembler and not object_store.contains(self.output_id):
-                with object_store.get(self.tree_id) as tree:
+                with object_store.get(self.tree_id) as tree, \
+                    object_store.new(self.output_id) as output_dir:
                     if not self.assembler.run(tree,
                                               build_tree,
                                               output_dir=output_dir,

test/__main__.py

@@ -43,7 +43,6 @@ if __name__ == '__main__':
     args = parser.parse_args()
 
     logging.info(f"Using {OBJECTS} for objects storage.")
-    logging.info(f"Using {OUTPUT_DIR} for output images storage.")
     logging.info(f"Using {OSBUILD} for building images.")
 
     f30_boot = IntegrationTestCase(

test/integration_tests/build.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import subprocess
 import sys
@@ -5,8 +6,8 @@ import sys
 from .config import *
 
 
-def run_osbuild(pipeline: str, build_pipeline: str, check=True):
+def run_osbuild(pipeline: str, build_pipeline: str):
-    cmd = OSBUILD + ["--store", OBJECTS, "-o", OUTPUT_DIR, pipeline]
+    cmd = OSBUILD + ["--json", "--store", OBJECTS, pipeline]
     if build_pipeline:
         cmd += ["--build-pipeline", build_pipeline]
     logging.info(f"Running osbuild: {cmd}")
@@ -17,10 +18,10 @@ def run_osbuild(pipeline: str, build_pipeline: str, check=True):
         print(osbuild.stderr.decode())
         print(f"{BOLD}STDOUT{RESET}")
         print(osbuild.stdout.decode())
-        if check:
+        sys.exit(1)
-            sys.exit(1)
 
-    return osbuild.returncode
+    result = json.loads(osbuild.stdout.decode())
+    return result["tree_id"], result.get("output_id")
 
 
 def build_testing_image(pipeline_full_path, build_pipeline_full_path):

test/integration_tests/config.py

@@ -7,5 +7,4 @@ RESET = "\033[0m"
 BOLD = "\033[1m"
 RED = "\033[31m"
 OBJECTS = os.environ.get("OBJECTS", ".osbuild-test")
-OUTPUT_DIR = os.environ.get("OUTPUT_DIR", "output-test")
 OSBUILD = os.environ.get("OSBUILD", "python3 -m osbuild --libdir .").split(' ')

test/integration_tests/run.py

@@ -11,7 +11,7 @@ def run_image(file_name: str):
     silence = ["-nographic", "-monitor", "none", "-serial", "none"]
     serial = ["-chardev", "stdio,id=stdio", "-device", "virtio-serial", "-device", "virtserialport,chardev=stdio"]
     cmd = ["qemu-system-x86_64", "-m", "1024", "-snapshot"] + \
-          acceleration + silence + serial + [f"{OUTPUT_DIR}/{file_name}"]
+          acceleration + silence + serial + [file_name]
     logging.info(f"Booting image: {cmd}")
     return subprocess.run(cmd, capture_output=True, timeout=EXPECTED_TIME_TO_BOOT, encoding="utf-8", check=True)
 
@@ -19,7 +19,7 @@ def run_image(file_name: str):
 @contextlib.contextmanager
 def extract_image(file_name: str):
     extract_dir = tempfile.mkdtemp(prefix="osbuild-")
-    archive = path.join(os.getcwd(), OUTPUT_DIR, file_name)
+    archive = path.join(os.getcwd(), file_name)
     subprocess.run(["tar", "xf", archive], cwd=extract_dir, check=True)
     try:
         yield extract_dir

test/integration_tests/test_case.py

@@ -5,6 +5,7 @@ from typing import List, Callable, Any
 from . import evaluate_test, rel_path
 from .build import run_osbuild
 from .run import run_image, extract_image
+from test.integration_tests.config import *
 
 
 class IntegrationTestType(Enum):
@@ -22,18 +23,18 @@ class IntegrationTestCase:
     type: IntegrationTestType
 
     def run(self):
-        run_osbuild(rel_path(f"pipelines/{self.pipeline}"), self.build_pipeline)
+        tree_id, output_id = run_osbuild(rel_path(f"pipelines/{self.pipeline}"), self.build_pipeline)
         if self.type == IntegrationTestType.BOOT_WITH_QEMU:
-            self.run_and_test()
+            self.run_and_test(output_id)
         else:
-            self.extract_and_test()
+            self.extract_and_test(output_id)
 
-    def run_and_test(self):
+    def run_and_test(self, output_id):
-        r = run_image(self.output_image)
+        r = run_image(f"{OBJECTS}/refs/{output_id}/{self.output_image}")
         for test in self.test_cases:
             evaluate_test(test, r.stdout)
 
-    def extract_and_test(self):
+    def extract_and_test(self, output_id):
-        with extract_image(self.output_image) as fstree:
+        with extract_image(f"{OBJECTS}/refs/{output_id}/{self.output_image}") as fstree:
             for test in self.test_cases:
                 evaluate_test(lambda: test(fstree), name=test.__name__)