osbuild: assemblers are pipelines now
Convert the assembler phase of the main pipeline in the old format into a new Pipeline that as the assembler as a stage, where the input of that stage is the main pipeline. This removes the need of having "assemblers" as special concepts and thus the corresponding code in `Pipeline` is removed. The new assembler pipeline is marked as exported, but the pipeline that builds the tree is not anymore. Adapt the `describe` and `output` functions of the `v1` format to handle the assembler pipeline. Also change the tests accordingly. NB: The id reported for the assembler via `--inspect` and the result will change as a result of this, since the assembler stage is now the first and only stage of a new pipeline and thus has no base anymore.
This commit is contained in:
Christian Kellner
parent
289d943d94
commit
53e9ec850b
4 changed files with 57 additions and 96 deletions
|
|
@ -29,14 +29,16 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
|
|||
stages = [describe_stage(s) for s in pipeline.stages]
|
||||
description["stages"] = stages
|
||||
|
||||
if pipeline.assembler:
|
||||
assembler = describe_stage(pipeline.assembler)
|
||||
description["assembler"] = assembler
|
||||
return description
|
||||
|
||||
description = {
|
||||
"pipeline": describe_pipeline(manifest["tree"])
|
||||
}
|
||||
pipeline = describe_pipeline(manifest["tree"])
|
||||
|
||||
assembler = manifest.get("assembler")
|
||||
if assembler:
|
||||
description = describe_stage(assembler.stages[0])
|
||||
pipeline["assembler"] = description
|
||||
|
||||
description = {"pipeline": pipeline}
|
||||
|
||||
if manifest.sources:
|
||||
description["sources"] = manifest.sources
|
||||
|
|
@ -44,6 +46,27 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
|
|||
return description
|
||||
|
||||
|
||||
def load_assembler(description: Dict, index: Index, manifest: Manifest):
|
||||
pipeline = manifest["tree"]
|
||||
|
||||
build, base, runner = pipeline.build, pipeline.tree_id, pipeline.runner
|
||||
name, options = description["name"], description.get("options", {})
|
||||
|
||||
# Add a pipeline with one stage for our assembler
|
||||
pipeline = manifest.add_pipeline("assembler", runner, build)
|
||||
pipeline.export = True
|
||||
|
||||
info = index.get_module_info("Assembler", name)
|
||||
|
||||
stage = pipeline.add_stage(info, options, {})
|
||||
info = index.get_module_info("Input", "org.osbuild.tree")
|
||||
stage.inputs = {
|
||||
"tree": Input(info, {"pipeline": {"id": base}})
|
||||
}
|
||||
|
||||
return pipeline
|
||||
|
||||
|
||||
def load_build(description: Dict, index: Index, manifest: Manifest, n: int):
|
||||
pipeline = description.get("pipeline")
|
||||
if pipeline:
|
||||
|
|
@ -77,16 +100,6 @@ def load_pipeline(description: Dict, index: Index, manifest: Manifest, n: int =
|
|||
info = index.get_module_info("Stage", s["name"])
|
||||
pipeline.add_stage(info, s.get("options", {}))
|
||||
|
||||
a = description.get("assembler")
|
||||
if a:
|
||||
info = index.get_module_info("Assembler", a["name"])
|
||||
asm = pipeline.set_assembler(info, a.get("options", {}))
|
||||
info = index.get_module_info("Input", "org.osbuild.tree")
|
||||
asm.inputs = {
|
||||
"tree": Input(info, {"pipeline": {"id": pipeline.tree_id}})
|
||||
}
|
||||
pipeline.export = True
|
||||
|
||||
return pipeline
|
||||
|
||||
|
||||
|
|
@ -100,6 +113,11 @@ def load(description: Dict, index: Index) -> Manifest:
|
|||
|
||||
load_pipeline(pipeline, index, manifest)
|
||||
|
||||
# load the assembler, if any
|
||||
assembler = pipeline.get("assembler")
|
||||
if assembler:
|
||||
load_assembler(assembler, index, manifest)
|
||||
|
||||
for pipeline in manifest.pipelines.values():
|
||||
for stage in pipeline.stages:
|
||||
stage.sources = sources
|
||||
|
|
@ -109,7 +127,8 @@ def load(description: Dict, index: Index) -> Manifest:
|
|||
|
||||
def get_ids(manifest: Manifest) -> Tuple[Optional[str], Optional[str]]:
|
||||
pipeline = manifest["tree"]
|
||||
return pipeline.tree_id, pipeline.output_id
|
||||
assembler = manifest.get("assembler")
|
||||
return pipeline.tree_id, assembler and assembler.tree_id
|
||||
|
||||
|
||||
def output(manifest: Manifest, res: Dict) -> Dict:
|
||||
|
|
@ -135,7 +154,15 @@ def output(manifest: Manifest, res: Dict) -> Dict:
|
|||
retval["assembler"] = assembler
|
||||
return retval
|
||||
|
||||
return result_for_pipeline(manifest["tree"])
|
||||
result = result_for_pipeline(manifest["tree"])
|
||||
|
||||
assembler = manifest.get("assembler")
|
||||
if assembler:
|
||||
current = res.get(assembler.id)
|
||||
if current:
|
||||
result["assembler"] = current["stages"][0]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def validate(manifest: Dict, index: Index) -> ValidationResult:
|
||||
|
|
|
|||
|
|
@ -122,16 +122,12 @@ class Pipeline:
|
|||
|
||||
@property
|
||||
def id(self):
|
||||
return self.output_id or self.tree_id
|
||||
return self.tree_id
|
||||
|
||||
@property
|
||||
def tree_id(self):
|
||||
return self.stages[-1].id if self.stages else None
|
||||
|
||||
@property
|
||||
def output_id(self):
|
||||
return self.assembler.id if self.assembler else None
|
||||
|
||||
def add_stage(self, info, options, sources_options=None):
|
||||
stage = Stage(info, sources_options, self.build, self.tree_id, options or {})
|
||||
self.stages.append(stage)
|
||||
|
|
@ -139,10 +135,6 @@ class Pipeline:
|
|||
self.assembler.base = stage.id
|
||||
return stage
|
||||
|
||||
def set_assembler(self, info, options=None):
|
||||
self.assembler = Stage(info, {}, self.build, self.tree_id, options or {})
|
||||
return self.assembler
|
||||
|
||||
def build_stages(self, object_store, monitor, libdir):
|
||||
results = {"success": True}
|
||||
|
||||
|
|
@ -220,39 +212,6 @@ class Pipeline:
|
|||
|
||||
return results, build_tree, tree
|
||||
|
||||
def assemble(self, object_store, build_tree, monitor, libdir):
|
||||
results = {"success": True}
|
||||
|
||||
if not self.assembler:
|
||||
return results, None
|
||||
|
||||
output = object_store.new()
|
||||
|
||||
with build_tree.read() as build_dir, \
|
||||
output.write() as output_dir:
|
||||
|
||||
monitor.assembler(self.assembler)
|
||||
|
||||
r = self.assembler.run(output_dir,
|
||||
self.runner,
|
||||
build_dir,
|
||||
object_store,
|
||||
monitor,
|
||||
libdir)
|
||||
|
||||
monitor.result(r)
|
||||
|
||||
results["assembler"] = r.as_dict()
|
||||
if not r.success:
|
||||
output.cleanup()
|
||||
results["success"] = False
|
||||
return results, None
|
||||
|
||||
if self.assembler.checkpoint:
|
||||
object_store.commit(output, self.assembler.id)
|
||||
|
||||
return results, output
|
||||
|
||||
def run(self, store, monitor, libdir, output_directory):
|
||||
results = {"success": True}
|
||||
|
||||
|
|
@ -263,21 +222,14 @@ class Pipeline:
|
|||
# tree exists, we return it as well, but we do not care if it is
|
||||
# missing, since it is not a mandatory part of the result and would
|
||||
# usually be needless overhead.
|
||||
obj = store.get(self.output_id)
|
||||
obj = store.get(self.tree_id)
|
||||
|
||||
if not obj:
|
||||
results, build_tree, _ = self.build_stages(store, monitor, libdir)
|
||||
results, _, obj = self.build_stages(store, monitor, libdir)
|
||||
|
||||
if not results["success"]:
|
||||
return results
|
||||
|
||||
r, obj = self.assemble(store,
|
||||
build_tree,
|
||||
monitor,
|
||||
libdir)
|
||||
|
||||
results.update(r) # This will also update 'success'
|
||||
|
||||
if self.export and obj:
|
||||
if output_directory:
|
||||
obj.export(output_directory)
|
||||
|
|
@ -322,17 +274,9 @@ class Manifest:
|
|||
stage.checkpoint = True
|
||||
points.remove(c)
|
||||
|
||||
def mark_assembler(assembler):
|
||||
c = assembler.id
|
||||
if c in points:
|
||||
assembler.checkpoint = True
|
||||
points.remove(c)
|
||||
|
||||
def mark_pipeline(pl):
|
||||
for stage in pl.stages:
|
||||
mark_stage(stage)
|
||||
if pl.assembler:
|
||||
mark_assembler(pl.assembler)
|
||||
|
||||
for pl in self.pipelines.values():
|
||||
mark_pipeline(pl)
|
||||
|
|
|
|||
|
|
@ -62,7 +62,6 @@ class TestFormatV1(unittest.TestCase):
|
|||
storedir = pathlib.Path(tmpdir, "store")
|
||||
monitor = NullMonitor(sys.stderr.fileno())
|
||||
libdir = os.path.abspath(os.curdir)
|
||||
print(libdir)
|
||||
store = ObjectStore(storedir)
|
||||
outdir = pathlib.Path(tmpdir, "out")
|
||||
outdir.mkdir()
|
||||
|
|
@ -94,7 +93,7 @@ class TestFormatV1(unittest.TestCase):
|
|||
# Load a pipeline and check the resulting manifest
|
||||
def check_stage(have: osbuild.Stage, want: Dict):
|
||||
self.assertEqual(have.name, want["name"])
|
||||
self.assertEqual(have.options, want["options"])
|
||||
self.assertEqual(have.options, want.get("options", {}))
|
||||
|
||||
index = osbuild.meta.Index(os.curdir)
|
||||
|
||||
|
|
@ -108,7 +107,7 @@ class TestFormatV1(unittest.TestCase):
|
|||
|
||||
# We have to have two build pipelines and a main pipeline
|
||||
self.assertTrue(manifest.pipelines)
|
||||
self.assertTrue(len(manifest.pipelines) == 3)
|
||||
self.assertTrue(len(manifest.pipelines) == 4)
|
||||
|
||||
# access the individual pipelines via their names
|
||||
|
||||
|
|
@ -131,17 +130,19 @@ class TestFormatV1(unittest.TestCase):
|
|||
|
||||
runner = build["runner"]
|
||||
|
||||
# main pipeline is the next one
|
||||
# the main, aka 'tree', pipeline
|
||||
pl = manifest["tree"]
|
||||
have = pl.stages[0]
|
||||
want = description["pipeline"]["stages"][0]
|
||||
self.assertEqual(pl.runner, runner)
|
||||
check_stage(have, want)
|
||||
|
||||
# the assembler
|
||||
have = pl.assembler
|
||||
# the assembler pipeline
|
||||
pl = manifest["assembler"]
|
||||
have = pl.stages[0]
|
||||
want = description["pipeline"]["assembler"]
|
||||
self.assertEqual(have.name, want["name"])
|
||||
self.assertEqual(pl.runner, runner)
|
||||
check_stage(have, want)
|
||||
|
||||
|
||||
def test_describe(self):
|
||||
|
|
|
|||
|
|
@ -38,10 +38,6 @@ class TapeMonitor(osbuild.monitor.BaseMonitor):
|
|||
self.counter["stages"] += 1
|
||||
self.stages.add(stage.id)
|
||||
|
||||
def assembler(self, assembler: osbuild.Stage):
|
||||
self.counter["assembler"] += 1
|
||||
self.asm = assembler.id
|
||||
|
||||
def result(self, result: osbuild.pipeline.BuildResult):
|
||||
self.counter["result"] += 1
|
||||
self.results.add(result.id)
|
||||
|
|
@ -63,8 +59,6 @@ class TestMonitor(unittest.TestCase):
|
|||
pipeline.add_stage(info, {
|
||||
"isthisthereallife": False
|
||||
})
|
||||
info = index.get_module_info("Assembler", "org.osbuild.noop")
|
||||
pipeline.set_assembler(info)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
storedir = os.path.join(tmpdir, "store")
|
||||
|
|
@ -84,7 +78,6 @@ class TestMonitor(unittest.TestCase):
|
|||
|
||||
assert res
|
||||
self.assertIn(pipeline.stages[0].id, log)
|
||||
self.assertIn(pipeline.assembler.id, log)
|
||||
self.assertIn("isthisthereallife", log)
|
||||
|
||||
@unittest.skipUnless(test.TestBase.can_bind_mount(), "root-only")
|
||||
|
|
@ -101,8 +94,6 @@ class TestMonitor(unittest.TestCase):
|
|||
pipeline.add_stage(noop_info, {
|
||||
"isthisjustfantasy": True
|
||||
})
|
||||
info = index.get_module_info("Assembler", "org.osbuild.noop")
|
||||
pipeline.set_assembler(info)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
storedir = os.path.join(tmpdir, "store")
|
||||
|
|
@ -119,10 +110,8 @@ class TestMonitor(unittest.TestCase):
|
|||
self.assertEqual(tape.counter["begin"], 1)
|
||||
self.assertEqual(tape.counter["finish"], 1)
|
||||
self.assertEqual(tape.counter["stages"], 2)
|
||||
self.assertEqual(tape.counter["assembler"], 1)
|
||||
self.assertEqual(tape.counter["stages"], 2)
|
||||
self.assertEqual(tape.counter["result"], 3)
|
||||
self.assertEqual(tape.counter["result"], 2)
|
||||
self.assertIn(pipeline.stages[0].id, tape.stages)
|
||||
self.assertIn(pipeline.assembler.id, tape.asm)
|
||||
self.assertIn("isthisthereallife", tape.output)
|
||||
self.assertIn("isthisjustfantasy", tape.output)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue