diff --git a/osbuild/formats/v1.py b/osbuild/formats/v1.py index 41b540ab..f0cd1f70 100644 --- a/osbuild/formats/v1.py +++ b/osbuild/formats/v1.py @@ -35,7 +35,7 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict: return description description = { - "pipeline": describe_pipeline(manifest.pipelines[-1]) + "pipeline": describe_pipeline(manifest["tree"]) } if manifest.sources: @@ -44,25 +44,34 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict: return description -def load_build(description: Dict, index: Index, manifest: Manifest): +def load_build(description: Dict, index: Index, manifest: Manifest, n: int): pipeline = description.get("pipeline") if pipeline: - build_pipeline = load_pipeline(pipeline, index, manifest) + build_pipeline = load_pipeline(pipeline, index, manifest, n + 1) else: build_pipeline = None return build_pipeline, description["runner"] -def load_pipeline(description: Dict, index: Index, manifest: Manifest) -> Pipeline: +def load_pipeline(description: Dict, index: Index, manifest: Manifest, n: int = 0) -> Pipeline: build = description.get("build") if build: - build_pipeline, runner = load_build(build, index, manifest) + build_pipeline, runner = load_build(build, index, manifest, n) else: build_pipeline, runner = None, detect_host_runner() + # the "main" pipeline is called `tree`, since it is building the + # tree that will later be used by the `assembler`. Nested build + # pipelines will get call "build", and "build-build-...", where + # the number of repetitions is equal their level of nesting + if not n: + name = "tree" + else: + name = "-".join(["build"] * n) + build_id = build_pipeline and build_pipeline.tree_id - pipeline = manifest.add_pipeline(runner, build_id) + pipeline = manifest.add_pipeline(name, runner, build_id) for s in description.get("stages", []): info = index.get_module_info("Stage", s["name"]) @@ -91,7 +100,7 @@ def load(description: Dict, index: Index) -> Manifest: load_pipeline(pipeline, index, manifest) - for pipeline in manifest.pipelines: + for pipeline in manifest.pipelines.values(): for stage in pipeline.stages: stage.sources = sources @@ -99,7 +108,7 @@ def load(description: Dict, index: Index) -> Manifest: def get_ids(manifest: Manifest) -> Tuple[Optional[str], Optional[str]]: - pipeline = manifest.pipelines[-1] + pipeline = manifest["tree"] return pipeline.tree_id, pipeline.output_id @@ -126,7 +135,7 @@ def output(manifest: Manifest, res: Dict) -> Dict: retval["assembler"] = assembler return retval - return result_for_pipeline(manifest.pipelines[-1]) + return result_for_pipeline(manifest["tree"]) def validate(manifest: Dict, index: Index) -> ValidationResult: diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index b145ea80..54c541c4 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -1,8 +1,9 @@ +import collections import contextlib import hashlib import json import os -from typing import Dict +from typing import Dict, Iterator from .api import API from . import buildroot @@ -111,7 +112,8 @@ class Stage: class Pipeline: - def __init__(self, runner=None, build=None): + def __init__(self, name: str, runner=None, build=None): + self.name = name self.build = build self.runner = runner self.stages = [] @@ -289,18 +291,20 @@ class Manifest: """Representation of a pipeline and its sources""" def __init__(self, source_options: Dict): - self.pipelines = [] + self.pipelines = collections.OrderedDict() self.sources = source_options - def add_pipeline(self, runner: str, build: str) -> Pipeline: - pipeline = Pipeline(runner, build) - self.pipelines.append(pipeline) + def add_pipeline(self, name: str, runner: str, build: str) -> Pipeline: + pipeline = Pipeline(name, runner, build) + if name in self.pipelines: + raise ValueError(f"Name {name} already exists") + self.pipelines[name] = pipeline return pipeline def build(self, store, monitor, libdir, output_directory): results = {"success": True} - for pl in self.pipelines: + for pl in self.pipelines.values(): res = pl.run(store, monitor, libdir, output_directory) results[pl.id] = res if not res["success"]: @@ -330,16 +334,22 @@ class Manifest: if pl.assembler: mark_assembler(pl.assembler) - for pl in self.pipelines: + for pl in self.pipelines.values(): mark_pipeline(pl) return points - def __getitem__(self, pipeline_id): - for pl in self.pipelines: - if pl.id == pipeline_id: + def __getitem__(self, name_or_id: str) -> Pipeline: + pl = self.pipelines.get(name_or_id) + if pl: + return pl + for pl in self.pipelines.values(): + if pl.id == name_or_id: return pl - raise KeyError("{pipeline_id} not found") + raise KeyError("{name_or_id} not found") + + def __iter__(self) -> Iterator[Pipeline]: + return iter(self.pipelines.values()) def detect_host_runner(): diff --git a/test/mod/test_fmt_v1.py b/test/mod/test_fmt_v1.py index 1791780d..d19a76c5 100644 --- a/test/mod/test_fmt_v1.py +++ b/test/mod/test_fmt_v1.py @@ -100,7 +100,7 @@ class TestFormatV1(unittest.TestCase): self.assertTrue(len(manifest.pipelines) == 2) build = description["pipeline"]["build"] - pl = manifest.pipelines[0] + pl = manifest["build"] have = pl.stages[0] want = build["pipeline"]["stages"][0] check_stage(have, want) @@ -108,7 +108,7 @@ class TestFormatV1(unittest.TestCase): runner = build["runner"] # main pipeline is the next one - pl = manifest.pipelines[1] + pl = manifest["tree"] have = pl.stages[0] want = description["pipeline"]["stages"][0] self.assertEqual(pl.runner, runner) diff --git a/test/mod/test_monitor.py b/test/mod/test_monitor.py index cb549769..0e6d90d0 100644 --- a/test/mod/test_monitor.py +++ b/test/mod/test_monitor.py @@ -58,7 +58,7 @@ class TestMonitor(unittest.TestCase): index = osbuild.meta.Index(os.curdir) runner = detect_host_runner() - pipeline = osbuild.Pipeline(runner=runner) + pipeline = osbuild.Pipeline("pipeline", runner=runner) info = index.get_module_info("Stage", "org.osbuild.noop") pipeline.add_stage(info, { "isthisthereallife": False @@ -93,7 +93,7 @@ class TestMonitor(unittest.TestCase): runner = detect_host_runner() index = osbuild.meta.Index(os.curdir) - pipeline = osbuild.Pipeline(runner=runner) + pipeline = osbuild.Pipeline("pipeline", runner=runner) noop_info = index.get_module_info("Stage", "org.osbuild.noop") pipeline.add_stage(noop_info, { "isthisthereallife": False