From 8ccc73d1c31f316756aee4bd65d76b129d01e018 Mon Sep 17 00:00:00 2001 From: Christian Kellner Date: Fri, 15 Jan 2021 18:34:28 +0100 Subject: [PATCH] pipeline assemblers are stages now Instead of using the `Assemblers` class to represent assemblers, use the `Stage` class: The `Pipeline.add_assembler` method will now instantiate and `Stage` instead of an `Assembler`. The tree that the pipeline built is converted to an Input (while loading the manifest description in `format/v1.py`) and all existing assemblers are converted to use that input as the tree input. The assembler run test is removed as the Assembler class itself is not used (i.e. run) anymore. --- assemblers/org.osbuild.noop | 4 +++- assemblers/org.osbuild.oci-archive | 4 +++- assemblers/org.osbuild.ostree.commit | 4 +++- assemblers/org.osbuild.qemu | 4 +++- assemblers/org.osbuild.rawfs | 4 +++- assemblers/org.osbuild.tar | 4 +++- osbuild/formats/v1.py | 8 +++++++- osbuild/pipeline.py | 19 +++++++------------ test/mod/test_monitor.py | 6 ++++-- test/mod/test_osbuild.py | 27 +++------------------------ 10 files changed, 39 insertions(+), 45 deletions(-) diff --git a/assemblers/org.osbuild.noop b/assemblers/org.osbuild.noop index 074e8d4d..94b85dae 100755 --- a/assemblers/org.osbuild.noop +++ b/assemblers/org.osbuild.noop @@ -24,5 +24,7 @@ def main(_tree, _output_dir, options): if __name__ == '__main__': args = osbuild.api.arguments() - r = main(args["tree"], args["output_dir"], args.get("options", {})) + args_input = args["inputs"]["tree"]["path"] + args_output = args["tree"] + r = main(args_input, args_output, args.get("options", {})) sys.exit(r) diff --git a/assemblers/org.osbuild.oci-archive b/assemblers/org.osbuild.oci-archive index 30fd5a20..c514c163 100755 --- a/assemblers/org.osbuild.oci-archive +++ b/assemblers/org.osbuild.oci-archive @@ -272,5 +272,7 @@ def main(tree, output_dir, options): if __name__ == '__main__': args = osbuild.api.arguments() - r = main(args["tree"], args["output_dir"], args["options"]) + args_input = args["inputs"]["tree"]["path"] + args_output = args["tree"] + r = main(args_input, args_output, args["options"]) sys.exit(r) diff --git a/assemblers/org.osbuild.ostree.commit b/assemblers/org.osbuild.ostree.commit index 182cf630..b97257ef 100755 --- a/assemblers/org.osbuild.ostree.commit +++ b/assemblers/org.osbuild.ostree.commit @@ -185,5 +185,7 @@ def main(tree, output_dir, options, meta): if __name__ == '__main__': args = api.arguments() - r = main(args["tree"], args["output_dir"], args["options"], args["meta"]) + args_input = args["inputs"]["tree"]["path"] + args_output = args["tree"] + r = main(args_input, args_output, args["options"], args["meta"]) sys.exit(r) diff --git a/assemblers/org.osbuild.qemu b/assemblers/org.osbuild.qemu index 9d6baf3f..e6bb648b 100755 --- a/assemblers/org.osbuild.qemu +++ b/assemblers/org.osbuild.qemu @@ -702,5 +702,7 @@ def main(tree, output_dir, options, loop_client): if __name__ == '__main__': args = osbuild.api.arguments() - ret = main(args["tree"], args["output_dir"], args["options"], remoteloop.LoopClient("/run/osbuild/api/remoteloop")) + args_input = args["inputs"]["tree"]["path"] + args_output = args["tree"] + ret = main(args_input, args_output, args["options"], remoteloop.LoopClient("/run/osbuild/api/remoteloop")) sys.exit(ret) diff --git a/assemblers/org.osbuild.rawfs b/assemblers/org.osbuild.rawfs index a5d5f54f..fd4f1c97 100755 --- a/assemblers/org.osbuild.rawfs +++ b/assemblers/org.osbuild.rawfs @@ -107,5 +107,7 @@ def main(tree, output_dir, options, loop_client): if __name__ == '__main__': args = osbuild.api.arguments() - r = main(args["tree"], args["output_dir"], args["options"], remoteloop.LoopClient("/run/osbuild/api/remoteloop")) + args_input = args["inputs"]["tree"]["path"] + args_output = args["tree"] + r = main(args_input, args_output, args["options"], remoteloop.LoopClient("/run/osbuild/api/remoteloop")) sys.exit(r) diff --git a/assemblers/org.osbuild.tar b/assemblers/org.osbuild.tar index 2d8e1006..108d4a34 100755 --- a/assemblers/org.osbuild.tar +++ b/assemblers/org.osbuild.tar @@ -80,5 +80,7 @@ def main(tree, output_dir, options): if __name__ == '__main__': args = osbuild.api.arguments() - r = main(args["tree"], args["output_dir"], args["options"]) + args_input = args["inputs"]["tree"]["path"] + args_output = args["tree"] + r = main(args_input, args_output, args["options"]) sys.exit(r) diff --git a/osbuild/formats/v1.py b/osbuild/formats/v1.py index 32f8e5aa..2413a5e0 100644 --- a/osbuild/formats/v1.py +++ b/osbuild/formats/v1.py @@ -2,6 +2,7 @@ from typing import Dict, List, Optional, Tuple from osbuild.meta import Index, ValidationResult +from ..inputs import Input from ..pipeline import Manifest, Pipeline, detect_host_runner @@ -69,7 +70,12 @@ def load_pipeline(description: Dict, index: Index, result: List[Pipeline]) -> Pi a = description.get("assembler") if a: - pipeline.set_assembler(a["name"], a.get("options", {})) + info = index.get_module_info("Assembler", a["name"]) + asm = pipeline.set_assembler(info, a.get("options", {})) + info = index.get_module_info("Input", "org.osbuild.tree") + asm.inputs = { + "tree": Input(info, {"pipeline": {"id": pipeline.tree_id}}) + } result.append(pipeline) diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index 45c61786..314d0ad8 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -187,8 +187,8 @@ class Pipeline: self.assembler.base = stage.id return stage - def set_assembler(self, name, options=None): - self.assembler = Assembler(name, self.build, self.tree_id, options or {}) + def set_assembler(self, info, options=None): + self.assembler = Stage(info, {}, self.build, self.tree_id, options or {}) return self.assembler def build_stages(self, object_store, monitor, libdir): @@ -268,7 +268,7 @@ class Pipeline: return results, build_tree, tree - def assemble(self, object_store, build_tree, tree, monitor, libdir): + def assemble(self, object_store, build_tree, monitor, libdir): results = {"success": True} if not self.assembler: @@ -277,18 +277,16 @@ class Pipeline: output = object_store.new() with build_tree.read() as build_dir, \ - tree.read() as input_dir, \ output.write() as output_dir: monitor.assembler(self.assembler) - r = self.assembler.run(input_dir, + r = self.assembler.run(output_dir, self.runner, build_dir, + object_store, monitor, - libdir, - output_dir, - var=object_store.store) + libdir) monitor.result(r) @@ -316,16 +314,13 @@ class Pipeline: obj = store.get(self.output_id) if not obj: - results, build_tree, tree = self.build_stages(store, - monitor, - libdir) + results, build_tree, _ = self.build_stages(store, monitor, libdir) if not results["success"]: return results r, obj = self.assemble(store, build_tree, - tree, monitor, libdir) diff --git a/test/mod/test_monitor.py b/test/mod/test_monitor.py index 7d7010b4..5f3e93c7 100644 --- a/test/mod/test_monitor.py +++ b/test/mod/test_monitor.py @@ -63,7 +63,8 @@ class TestMonitor(unittest.TestCase): pipeline.add_stage(info, { "isthisthereallife": False }) - pipeline.set_assembler("org.osbuild.noop") + info = index.get_module_info("Assembler", "org.osbuild.noop") + pipeline.set_assembler(info) with tempfile.TemporaryDirectory() as tmpdir: storedir = os.path.join(tmpdir, "store") @@ -100,7 +101,8 @@ class TestMonitor(unittest.TestCase): pipeline.add_stage(noop_info, { "isthisjustfantasy": True }) - pipeline.set_assembler("org.osbuild.noop") + info = index.get_module_info("Assembler", "org.osbuild.noop") + pipeline.set_assembler(info) with tempfile.TemporaryDirectory() as tmpdir: storedir = os.path.join(tmpdir, "store") diff --git a/test/mod/test_osbuild.py b/test/mod/test_osbuild.py index 0b5bdf79..b1db3757 100644 --- a/test/mod/test_osbuild.py +++ b/test/mod/test_osbuild.py @@ -60,28 +60,6 @@ class TestDescriptions(unittest.TestCase): self.assertEqual(res.success, True) self.assertEqual(res.id, stage.id) - @unittest.skipUnless(test.TestBase.can_bind_mount(), "root-only") - def test_assembler_run(self): - asm = osbuild.Assembler("org.osbuild.noop", None, None, {}) - - with tempfile.TemporaryDirectory() as tmpdir: - - data = pathlib.Path(tmpdir, "data") - cache = pathlib.Path(tmpdir, "cache") - output = pathlib.Path(tmpdir, "output") - root = pathlib.Path("/") - runner = detect_host_runner() - monitor = NullMonitor(sys.stderr.fileno()) - libdir = os.path.abspath(os.curdir) - - for p in [data, cache, output]: - p.mkdir() - - res = asm.run(data, runner, root, monitor, libdir, output) - - self.assertEqual(res.success, True) - self.assertEqual(res.id, asm.id) - def test_pipeline(self): index = osbuild.meta.Index(os.curdir) @@ -91,7 +69,8 @@ class TestDescriptions(unittest.TestCase): pipeline = osbuild.Pipeline("org.osbuild.test", build.tree_id) pipeline.add_stage(test_info, {"one": 2}) - pipeline.set_assembler("org.osbuild.test") + info = index.get_module_info("Assembler", "org.osbuild.noop") + pipeline.set_assembler(info) manifest = osbuild.Manifest([build, pipeline], {}) @@ -115,7 +94,7 @@ class TestDescriptions(unittest.TestCase): } ], "assembler": { - "name": "org.osbuild.test" + "name": "org.osbuild.noop" } } })