pipeline: identify pipelines by name

Every pipeline that gets added to the `Manifest` now need to have
a unique name by which it can be identified. The version 1 format
loader is changed so that the main pipeline that builds the tree
is always called `tree`. The build pipeline for it will be called
`build` and further recursive build pipelines `build-build`, where
the number of repetitions of `build` corresponds to their level of
nesting. An assembler, if it exists, will be added as `assembler`.
The `Manifest.__getitem__` helper is changed so it will first try
to access pipeline via its name and then fall back to an id based
search. NB: in the degenrate case of multiple pipelines that have
exactly the same `id`, i.e. same stages, with the same options and
same build pipeline, only the first one will be return; but only
the first one here will be built as well, so this is in practice
not a problem.
The formatter uses this helper to get the tree pipeline  via its
name wherever it is needed.
This also adds an `__iter__` method `Manifest` to ease iterating
over just the pipeline values, a la `for pipeline in manifet`.

osbuild/formats/v1.py

@@ -35,7 +35,7 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
         return description
 
     description = {
-        "pipeline": describe_pipeline(manifest.pipelines[-1])
+        "pipeline": describe_pipeline(manifest["tree"])
     }
 
     if manifest.sources:
@@ -44,25 +44,34 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
     return description
 
 
-def load_build(description: Dict, index: Index, manifest: Manifest):
+def load_build(description: Dict, index: Index, manifest: Manifest, n: int):
     pipeline = description.get("pipeline")
     if pipeline:
-        build_pipeline = load_pipeline(pipeline, index, manifest)
+        build_pipeline = load_pipeline(pipeline, index, manifest, n + 1)
     else:
         build_pipeline = None
 
     return build_pipeline, description["runner"]
 
 
-def load_pipeline(description: Dict, index: Index, manifest: Manifest) -> Pipeline:
+def load_pipeline(description: Dict, index: Index, manifest: Manifest, n: int = 0) -> Pipeline:
     build = description.get("build")
     if build:
-        build_pipeline, runner = load_build(build, index, manifest)
+        build_pipeline, runner = load_build(build, index, manifest, n)
     else:
         build_pipeline, runner = None, detect_host_runner()
 
+    # the "main" pipeline is called `tree`, since it is building the
+    # tree that will later be used by the `assembler`. Nested build
+    # pipelines will get call "build", and "build-build-...", where
+    # the number of repetitions is equal their level of nesting
+    if not n:
+        name = "tree"
+    else:
+        name = "-".join(["build"] * n)
+
     build_id = build_pipeline and build_pipeline.tree_id
-    pipeline = manifest.add_pipeline(runner, build_id)
+    pipeline = manifest.add_pipeline(name, runner, build_id)
 
     for s in description.get("stages", []):
         info = index.get_module_info("Stage", s["name"])
@@ -91,7 +100,7 @@ def load(description: Dict, index: Index) -> Manifest:
 
     load_pipeline(pipeline, index, manifest)
 
-    for pipeline in manifest.pipelines:
+    for pipeline in manifest.pipelines.values():
         for stage in pipeline.stages:
             stage.sources = sources
 
@@ -99,7 +108,7 @@ def load(description: Dict, index: Index) -> Manifest:
 
 
 def get_ids(manifest: Manifest) -> Tuple[Optional[str], Optional[str]]:
-    pipeline = manifest.pipelines[-1]
+    pipeline = manifest["tree"]
     return pipeline.tree_id, pipeline.output_id
 
 
@@ -126,7 +135,7 @@ def output(manifest: Manifest, res: Dict) -> Dict:
             retval["assembler"] = assembler
         return retval
 
-    return result_for_pipeline(manifest.pipelines[-1])
+    return result_for_pipeline(manifest["tree"])
 
 
 def validate(manifest: Dict, index: Index) -> ValidationResult:

osbuild/pipeline.py

@@ -1,8 +1,9 @@
+import collections
 import contextlib
 import hashlib
 import json
 import os
-from typing import Dict
+from typing import Dict, Iterator
 
 from .api import API
 from . import buildroot
@@ -111,7 +112,8 @@ class Stage:
 
 
 class Pipeline:
-    def __init__(self, runner=None, build=None):
+    def __init__(self, name: str, runner=None, build=None):
+        self.name = name
         self.build = build
         self.runner = runner
         self.stages = []
@@ -289,18 +291,20 @@ class Manifest:
     """Representation of a pipeline and its sources"""
 
     def __init__(self, source_options: Dict):
-        self.pipelines = []
+        self.pipelines = collections.OrderedDict()
         self.sources = source_options
 
-    def add_pipeline(self, runner: str, build: str) -> Pipeline:
-        pipeline = Pipeline(runner, build)
-        self.pipelines.append(pipeline)
+    def add_pipeline(self, name: str, runner: str, build: str) -> Pipeline:
+        pipeline = Pipeline(name, runner, build)
+        if name in self.pipelines:
+            raise ValueError(f"Name {name} already exists")
+        self.pipelines[name] = pipeline
         return pipeline
 
     def build(self, store, monitor, libdir, output_directory):
         results = {"success": True}
 
-        for pl in self.pipelines:
+        for pl in self.pipelines.values():
             res = pl.run(store, monitor, libdir, output_directory)
             results[pl.id] = res
             if not res["success"]:
@@ -330,16 +334,22 @@ class Manifest:
             if pl.assembler:
                 mark_assembler(pl.assembler)
 
-        for pl in self.pipelines:
+        for pl in self.pipelines.values():
             mark_pipeline(pl)
 
         return points
 
-    def __getitem__(self, pipeline_id):
-        for pl in self.pipelines:
-            if pl.id == pipeline_id:
+    def __getitem__(self, name_or_id: str) -> Pipeline:
+        pl = self.pipelines.get(name_or_id)
+        if pl:
+            return pl
+        for pl in self.pipelines.values():
+            if pl.id == name_or_id:
                 return pl
-        raise KeyError("{pipeline_id} not found")
+        raise KeyError("{name_or_id} not found")
+
+    def __iter__(self) -> Iterator[Pipeline]:
+        return iter(self.pipelines.values())
 
 
 def detect_host_runner():

test/mod/test_fmt_v1.py

@@ -100,7 +100,7 @@ class TestFormatV1(unittest.TestCase):
         self.assertTrue(len(manifest.pipelines) == 2)
 
         build = description["pipeline"]["build"]
-        pl = manifest.pipelines[0]
+        pl = manifest["build"]
         have = pl.stages[0]
         want = build["pipeline"]["stages"][0]
         check_stage(have, want)
@@ -108,7 +108,7 @@ class TestFormatV1(unittest.TestCase):
         runner = build["runner"]
 
         # main pipeline is the next one
-        pl = manifest.pipelines[1]
+        pl = manifest["tree"]
         have = pl.stages[0]
         want = description["pipeline"]["stages"][0]
         self.assertEqual(pl.runner, runner)

test/mod/test_monitor.py

@@ -58,7 +58,7 @@ class TestMonitor(unittest.TestCase):
         index = osbuild.meta.Index(os.curdir)
 
         runner = detect_host_runner()
-        pipeline = osbuild.Pipeline(runner=runner)
+        pipeline = osbuild.Pipeline("pipeline", runner=runner)
         info = index.get_module_info("Stage", "org.osbuild.noop")
         pipeline.add_stage(info, {
             "isthisthereallife": False
@@ -93,7 +93,7 @@ class TestMonitor(unittest.TestCase):
         runner = detect_host_runner()
         index = osbuild.meta.Index(os.curdir)
 
-        pipeline = osbuild.Pipeline(runner=runner)
+        pipeline = osbuild.Pipeline("pipeline", runner=runner)
         noop_info = index.get_module_info("Stage", "org.osbuild.noop")
         pipeline.add_stage(noop_info, {
             "isthisthereallife": False