diff --git a/osbuild/formats/v1.py b/osbuild/formats/v1.py index f0cd1f70..578afe28 100644 --- a/osbuild/formats/v1.py +++ b/osbuild/formats/v1.py @@ -29,14 +29,16 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict: stages = [describe_stage(s) for s in pipeline.stages] description["stages"] = stages - if pipeline.assembler: - assembler = describe_stage(pipeline.assembler) - description["assembler"] = assembler return description - description = { - "pipeline": describe_pipeline(manifest["tree"]) - } + pipeline = describe_pipeline(manifest["tree"]) + + assembler = manifest.get("assembler") + if assembler: + description = describe_stage(assembler.stages[0]) + pipeline["assembler"] = description + + description = {"pipeline": pipeline} if manifest.sources: description["sources"] = manifest.sources @@ -44,6 +46,27 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict: return description +def load_assembler(description: Dict, index: Index, manifest: Manifest): + pipeline = manifest["tree"] + + build, base, runner = pipeline.build, pipeline.tree_id, pipeline.runner + name, options = description["name"], description.get("options", {}) + + # Add a pipeline with one stage for our assembler + pipeline = manifest.add_pipeline("assembler", runner, build) + pipeline.export = True + + info = index.get_module_info("Assembler", name) + + stage = pipeline.add_stage(info, options, {}) + info = index.get_module_info("Input", "org.osbuild.tree") + stage.inputs = { + "tree": Input(info, {"pipeline": {"id": base}}) + } + + return pipeline + + def load_build(description: Dict, index: Index, manifest: Manifest, n: int): pipeline = description.get("pipeline") if pipeline: @@ -77,16 +100,6 @@ def load_pipeline(description: Dict, index: Index, manifest: Manifest, n: int = info = index.get_module_info("Stage", s["name"]) pipeline.add_stage(info, s.get("options", {})) - a = description.get("assembler") - if a: - info = index.get_module_info("Assembler", a["name"]) - asm = pipeline.set_assembler(info, a.get("options", {})) - info = index.get_module_info("Input", "org.osbuild.tree") - asm.inputs = { - "tree": Input(info, {"pipeline": {"id": pipeline.tree_id}}) - } - pipeline.export = True - return pipeline @@ -100,6 +113,11 @@ def load(description: Dict, index: Index) -> Manifest: load_pipeline(pipeline, index, manifest) + # load the assembler, if any + assembler = pipeline.get("assembler") + if assembler: + load_assembler(assembler, index, manifest) + for pipeline in manifest.pipelines.values(): for stage in pipeline.stages: stage.sources = sources @@ -109,7 +127,8 @@ def load(description: Dict, index: Index) -> Manifest: def get_ids(manifest: Manifest) -> Tuple[Optional[str], Optional[str]]: pipeline = manifest["tree"] - return pipeline.tree_id, pipeline.output_id + assembler = manifest.get("assembler") + return pipeline.tree_id, assembler and assembler.tree_id def output(manifest: Manifest, res: Dict) -> Dict: @@ -135,7 +154,15 @@ def output(manifest: Manifest, res: Dict) -> Dict: retval["assembler"] = assembler return retval - return result_for_pipeline(manifest["tree"]) + result = result_for_pipeline(manifest["tree"]) + + assembler = manifest.get("assembler") + if assembler: + current = res.get(assembler.id) + if current: + result["assembler"] = current["stages"][0] + + return result def validate(manifest: Dict, index: Index) -> ValidationResult: diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index 7c27c52e..6c1843d7 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -122,16 +122,12 @@ class Pipeline: @property def id(self): - return self.output_id or self.tree_id + return self.tree_id @property def tree_id(self): return self.stages[-1].id if self.stages else None - @property - def output_id(self): - return self.assembler.id if self.assembler else None - def add_stage(self, info, options, sources_options=None): stage = Stage(info, sources_options, self.build, self.tree_id, options or {}) self.stages.append(stage) @@ -139,10 +135,6 @@ class Pipeline: self.assembler.base = stage.id return stage - def set_assembler(self, info, options=None): - self.assembler = Stage(info, {}, self.build, self.tree_id, options or {}) - return self.assembler - def build_stages(self, object_store, monitor, libdir): results = {"success": True} @@ -220,39 +212,6 @@ class Pipeline: return results, build_tree, tree - def assemble(self, object_store, build_tree, monitor, libdir): - results = {"success": True} - - if not self.assembler: - return results, None - - output = object_store.new() - - with build_tree.read() as build_dir, \ - output.write() as output_dir: - - monitor.assembler(self.assembler) - - r = self.assembler.run(output_dir, - self.runner, - build_dir, - object_store, - monitor, - libdir) - - monitor.result(r) - - results["assembler"] = r.as_dict() - if not r.success: - output.cleanup() - results["success"] = False - return results, None - - if self.assembler.checkpoint: - object_store.commit(output, self.assembler.id) - - return results, output - def run(self, store, monitor, libdir, output_directory): results = {"success": True} @@ -263,21 +222,14 @@ class Pipeline: # tree exists, we return it as well, but we do not care if it is # missing, since it is not a mandatory part of the result and would # usually be needless overhead. - obj = store.get(self.output_id) + obj = store.get(self.tree_id) if not obj: - results, build_tree, _ = self.build_stages(store, monitor, libdir) + results, _, obj = self.build_stages(store, monitor, libdir) if not results["success"]: return results - r, obj = self.assemble(store, - build_tree, - monitor, - libdir) - - results.update(r) # This will also update 'success' - if self.export and obj: if output_directory: obj.export(output_directory) @@ -322,17 +274,9 @@ class Manifest: stage.checkpoint = True points.remove(c) - def mark_assembler(assembler): - c = assembler.id - if c in points: - assembler.checkpoint = True - points.remove(c) - def mark_pipeline(pl): for stage in pl.stages: mark_stage(stage) - if pl.assembler: - mark_assembler(pl.assembler) for pl in self.pipelines.values(): mark_pipeline(pl) diff --git a/test/mod/test_fmt_v1.py b/test/mod/test_fmt_v1.py index 84b00096..62a1c6bf 100644 --- a/test/mod/test_fmt_v1.py +++ b/test/mod/test_fmt_v1.py @@ -62,7 +62,6 @@ class TestFormatV1(unittest.TestCase): storedir = pathlib.Path(tmpdir, "store") monitor = NullMonitor(sys.stderr.fileno()) libdir = os.path.abspath(os.curdir) - print(libdir) store = ObjectStore(storedir) outdir = pathlib.Path(tmpdir, "out") outdir.mkdir() @@ -94,7 +93,7 @@ class TestFormatV1(unittest.TestCase): # Load a pipeline and check the resulting manifest def check_stage(have: osbuild.Stage, want: Dict): self.assertEqual(have.name, want["name"]) - self.assertEqual(have.options, want["options"]) + self.assertEqual(have.options, want.get("options", {})) index = osbuild.meta.Index(os.curdir) @@ -108,7 +107,7 @@ class TestFormatV1(unittest.TestCase): # We have to have two build pipelines and a main pipeline self.assertTrue(manifest.pipelines) - self.assertTrue(len(manifest.pipelines) == 3) + self.assertTrue(len(manifest.pipelines) == 4) # access the individual pipelines via their names @@ -131,17 +130,19 @@ class TestFormatV1(unittest.TestCase): runner = build["runner"] - # main pipeline is the next one + # the main, aka 'tree', pipeline pl = manifest["tree"] have = pl.stages[0] want = description["pipeline"]["stages"][0] self.assertEqual(pl.runner, runner) check_stage(have, want) - # the assembler - have = pl.assembler + # the assembler pipeline + pl = manifest["assembler"] + have = pl.stages[0] want = description["pipeline"]["assembler"] - self.assertEqual(have.name, want["name"]) + self.assertEqual(pl.runner, runner) + check_stage(have, want) def test_describe(self): diff --git a/test/mod/test_monitor.py b/test/mod/test_monitor.py index 0e6d90d0..3816e074 100644 --- a/test/mod/test_monitor.py +++ b/test/mod/test_monitor.py @@ -38,10 +38,6 @@ class TapeMonitor(osbuild.monitor.BaseMonitor): self.counter["stages"] += 1 self.stages.add(stage.id) - def assembler(self, assembler: osbuild.Stage): - self.counter["assembler"] += 1 - self.asm = assembler.id - def result(self, result: osbuild.pipeline.BuildResult): self.counter["result"] += 1 self.results.add(result.id) @@ -63,8 +59,6 @@ class TestMonitor(unittest.TestCase): pipeline.add_stage(info, { "isthisthereallife": False }) - info = index.get_module_info("Assembler", "org.osbuild.noop") - pipeline.set_assembler(info) with tempfile.TemporaryDirectory() as tmpdir: storedir = os.path.join(tmpdir, "store") @@ -84,7 +78,6 @@ class TestMonitor(unittest.TestCase): assert res self.assertIn(pipeline.stages[0].id, log) - self.assertIn(pipeline.assembler.id, log) self.assertIn("isthisthereallife", log) @unittest.skipUnless(test.TestBase.can_bind_mount(), "root-only") @@ -101,8 +94,6 @@ class TestMonitor(unittest.TestCase): pipeline.add_stage(noop_info, { "isthisjustfantasy": True }) - info = index.get_module_info("Assembler", "org.osbuild.noop") - pipeline.set_assembler(info) with tempfile.TemporaryDirectory() as tmpdir: storedir = os.path.join(tmpdir, "store") @@ -119,10 +110,8 @@ class TestMonitor(unittest.TestCase): self.assertEqual(tape.counter["begin"], 1) self.assertEqual(tape.counter["finish"], 1) self.assertEqual(tape.counter["stages"], 2) - self.assertEqual(tape.counter["assembler"], 1) self.assertEqual(tape.counter["stages"], 2) - self.assertEqual(tape.counter["result"], 3) + self.assertEqual(tape.counter["result"], 2) self.assertIn(pipeline.stages[0].id, tape.stages) - self.assertIn(pipeline.assembler.id, tape.asm) self.assertIn("isthisthereallife", tape.output) self.assertIn("isthisjustfantasy", tape.output)