osbuild: flatten the pipeline

Instead of having build pipelines nested within the pipeline it is
the build pipeline for, the nested structure is transferred into a
flat list of pipelines. As a result the recursion is gone and all
the pipelines and trees are build one after the other. This is now
possible since floating objects are kept alive by the store itself
and all trees that are being built are transparently via them.
The immediate result dictionary changed accordingly. To keep the
JSON output of osbuild the same, the result is now routed through
a format specific converter.
Additionally, the v1 format module gained a function to retrieve
the global tree_id and output_id. With the new models those global
ids will go away eventually and thus need to go through the format
specific code.

osbuild/formats/v1.py

@@ -1,6 +1,6 @@
 # Version 1 of the manifest description
 
-from typing import Dict
+from typing import Dict, List, Optional, Tuple
 from osbuild.meta import Index, ValidationResult
 from ..pipeline import Manifest, Pipeline, detect_host_runner
 
@@ -18,7 +18,7 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
     def describe_pipeline(pipeline: Pipeline) -> Dict:
         description = {}
         if pipeline.build:
-            build = pipeline.build
+            build = manifest[pipeline.build]
             description["build"] = {
                 "pipeline": describe_pipeline(build),
                 "runner": pipeline.runner
@@ -34,7 +34,7 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
         return description
 
     description = {
-        "pipeline": describe_pipeline(manifest.pipeline)
+        "pipeline": describe_pipeline(manifest.pipelines[-1])
     }
 
     if manifest.sources:
@@ -43,24 +43,25 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
     return description
 
 
-def load_build(description: Dict, sources_options: Dict):
+def load_build(description: Dict, sources_options: Dict, result: List[Pipeline]):
     pipeline = description.get("pipeline")
     if pipeline:
-        build_pipeline = load_pipeline(pipeline, sources_options)
+        build_pipeline = load_pipeline(pipeline, sources_options, result)
     else:
         build_pipeline = None
 
     return build_pipeline, description["runner"]
 
 
-def load_pipeline(description: Dict, sources_options: Dict) -> Pipeline:
+def load_pipeline(description: Dict, sources_options: Dict, result: List[Pipeline]) -> Pipeline:
     build = description.get("build")
     if build:
-        build_pipeline, runner = load_build(build, sources_options)
+        build_pipeline, runner = load_build(build, sources_options, result)
     else:
         build_pipeline, runner = None, detect_host_runner()
 
-    pipeline = Pipeline(runner, build_pipeline)
+
+    pipeline = Pipeline(runner, build_pipeline and build_pipeline.tree_id)
 
     for s in description.get("stages", []):
         pipeline.add_stage(s["name"], sources_options, s.get("options", {}))
@@ -69,6 +70,8 @@ def load_pipeline(description: Dict, sources_options: Dict) -> Pipeline:
     if a:
         pipeline.set_assembler(a["name"], a.get("options", {}))
 
+    result.append(pipeline)
+
     return pipeline
 
 
@@ -78,12 +81,40 @@ def load(description: Dict) -> Manifest:
     pipeline = description.get("pipeline", {})
     sources = description.get("sources", {})
 
-    pipeline = load_pipeline(pipeline, sources)
-    manifest = Manifest(pipeline, sources)
+    pipelines = []
+
+    load_pipeline(pipeline, sources, pipelines)
+
+    manifest = Manifest(pipelines, sources)
 
     return manifest
 
 
+def get_ids(manifest: Manifest) -> Tuple[Optional[str], Optional[str]]:
+    pipeline = manifest.pipelines[-1]
+    return pipeline.tree_id, pipeline.output_id
+
+
+def output(manifest: Manifest, res: Dict) -> Dict:
+    """Convert a result into the v1 format"""
+
+    def result_for_pipeline(pipeline):
+        current = res[pipeline.id]
+        retval = {"success": current["success"]}
+        if pipeline.build:
+            build = manifest[pipeline.build]
+            retval["build"] = result_for_pipeline(build)
+        stages = current.get("stages")
+        if stages:
+            retval["stages"] = stages
+        assembler = current.get("assembler")
+        if assembler:
+            retval["assembler"] = assembler
+        return retval
+
+    return result_for_pipeline(manifest.pipelines[-1])
+
+
 def validate(manifest: Dict, index: Index) -> ValidationResult:
     """Validate a OSBuild manifest
 

osbuild/main_cli.py

@@ -91,7 +91,6 @@ def osbuild_cli():
         return 2
 
     manifest = fmt.load(desc)
-    pipeline = manifest.pipeline
 
     if args.checkpoint:
         missed = manifest.mark_checkpoints(args.checkpoint)
@@ -128,12 +127,14 @@ def osbuild_cli():
         return 130
 
     if args.json:
+        r = fmt.output(manifest, r)
         json.dump(r, sys.stdout)
         sys.stdout.write("\n")
     else:
         if r["success"]:
-            print("tree id:", pipeline.tree_id)
-            print("output id:", pipeline.output_id)
+            tree_id, output_id = fmt.get_ids(manifest)
+            print("tree id:", tree_id)
+            print("output id:", output_id)
         else:
             print()
             print(f"{RESET}{BOLD}{RED}Failed{RESET}")

osbuild/pipeline.py

@@ -3,7 +3,7 @@ import hashlib
 import json
 import os
 import tempfile
-from typing import Dict
+from typing import Dict, List
 
 from .api import API
 from . import buildroot
@@ -160,15 +160,13 @@ class Pipeline:
         return self.assembler.id if self.assembler else None
 
     def add_stage(self, name, sources_options=None, options=None):
-        build = self.build.tree_id if self.build else None
-        stage = Stage(name, sources_options, build, self.tree_id, options or {})
+        stage = Stage(name, sources_options, self.build, self.tree_id, options or {})
         self.stages.append(stage)
         if self.assembler:
             self.assembler.base = stage.id
 
     def set_assembler(self, name, options=None):
-        build = self.build.tree_id if self.build else None
-        self.assembler = Assembler(name, build, self.tree_id, options or {})
+        self.assembler = Assembler(name, self.build, self.tree_id, options or {})
 
     def build_stages(self, object_store, monitor, libdir):
         results = {"success": True}
@@ -182,22 +180,10 @@ class Pipeline:
         if not self.build:
             build_tree = objectstore.HostTree(object_store)
         else:
-            build = self.build
+            build_tree = object_store.get(self.build)
 
-            r, t, tree = build.build_stages(object_store,
-                                            monitor,
-                                            libdir)
-
-            results["build"] = r
-            if not r["success"]:
-                results["success"] = False
-                return results, None, None
-
-            # Cleanup the build tree (`t`) which was used to
-            # build `tree`; it is now not needed anymore
-            t.cleanup()
-
-            build_tree = tree
+        if not build_tree:
+            raise AssertionError(f"build tree {self.build} not found")
 
         # If there are no stages, just return build tree we just
         # obtained and a new, clean `tree`
@@ -335,13 +321,21 @@ class Pipeline:
 class Manifest:
     """Representation of a pipeline and its sources"""
 
-    def __init__(self, pipeline: Pipeline, source_options: Dict):
-        self.pipeline = pipeline
+    def __init__(self, pipelines: List[Pipeline], source_options: Dict):
+        self.pipelines = pipelines
         self.sources = source_options
 
     def build(self, store, monitor, libdir, output_directory):
-        return self.pipeline.run(store, monitor, libdir, output_directory)
+        results = {"success": True}
 
+        for pl in self.pipelines:
+            res = pl.run(store, monitor, libdir, output_directory)
+            results[pl.id] = res
+            if not res["success"]:
+                results["success"] = False
+                return results
+
+        return results
 
     def mark_checkpoints(self, checkpoints):
         points = set(checkpoints)
@@ -363,12 +357,18 @@ class Manifest:
                 mark_stage(stage)
             if pl.assembler:
                 mark_assembler(pl.assembler)
-            if pl.build:
-                mark_pipeline(pl.build)
 
-        mark_pipeline(self.pipeline)
+        for pl in self.pipelines:
+            mark_pipeline(pl)
+
         return points
 
+    def __getitem__(self, pipeline_id):
+        for pl in self.pipelines:
+            if pl.id == pipeline_id:
+                return pl
+        raise KeyError("{pipeline_id} not found")
+
 
 def detect_host_runner():
     """Use os-release(5) to detect the runner for the host"""

test/mod/test_osbuild.py

@@ -82,11 +82,11 @@ class TestDescriptions(unittest.TestCase):
         build = osbuild.Pipeline("org.osbuild.test")
         build.add_stage("org.osbuild.test", {}, {"one": 1})
 
-        pipeline = osbuild.Pipeline("org.osbuild.test", build)
+        pipeline = osbuild.Pipeline("org.osbuild.test", build.tree_id)
         pipeline.add_stage("org.osbuild.test", {}, {"one": 2})
         pipeline.set_assembler("org.osbuild.test")
 
-        manifest = osbuild.Manifest(pipeline, {})
+        manifest = osbuild.Manifest([build, pipeline], {})
 
         self.assertEqual(fmt.describe(manifest), {
             "pipeline": {

test/test.py

@@ -355,7 +355,8 @@ class OSBuild(contextlib.AbstractContextManager):
         manifest_json = json.loads(manifest_data)
 
         manifest = fmt.load(manifest_json)
-        return manifest.pipeline.tree_id
+        tree_id, _ = fmt.get_ids(manifest)
+        return tree_id
 
     @contextlib.contextmanager
     def map_object(self, obj):