diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index b52d7cd4..abe5398f 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -3,7 +3,7 @@ import contextlib import hashlib import json import os -from typing import Dict, Iterator, List, Optional +from typing import Dict, Generator, Iterable, Iterator, List, Optional from .api import API from . import buildroot @@ -64,6 +64,18 @@ class Stage: m.update(json.dumps(data, sort_keys=True).encode()) return m.hexdigest() + @property + def dependencies(self) -> Generator[str, None, None]: + """Return a list of pipeline ids this stage depends on""" + + for ip in self.inputs.values(): + + if ip.origin != "org.osbuild.pipeline": + continue + + for ref in ip.refs: + yield ref + def add_input(self, name, info, origin, options=None): ip = Input(name, info, origin, options or {}) self.inputs[name] = ip @@ -344,6 +356,55 @@ class Manifest: for source in self.sources: source.download(mgr, store, libdir) + def depsolve(self, store, targets: Iterable[str]) -> List[str]: + """Return the list of pipelines that need to be built + + Given a list of target pipelines, return the names + of all pipelines and their dependencies that are not + already present in the store. + """ + + # A stack of pipelines to check if they need to be built + check = list(map(self.get, targets)) + + # The ordered result "set", will be reversed at the end + build = collections.OrderedDict() + + while check: + pl = check.pop() # get the last(!) item + + if store.contains(pl.id): + continue + + # The store does not have this pipeline, it needs to + # be built, add it to the ordered result set and + # ensure it is at the end, i.e. built before previously + # checked items. NB: the result set is reversed before + # it gets returned. This ensures that a dependency that + # gets checked multiple times, like a build pipeline, + # always gets built before its dependent pipeline. + build[pl.id] = pl + build.move_to_end(pl.id) + + # Add all dependencies to the stack of things to check, + # starting with the build pipeline, if there is one + if pl.build: + check.append(self.get(pl.build)) + + # Stages depend on other pipeline via pipeline inputs. + # We check in reversed order until we hit a checkpoint + for stage in reversed(pl.stages): + + # we stop if we have a checkpoint, i.e. we don't + # need to build any stages after that checkpoint + if store.contains(stage.id): + break + + pls = map(self.get, stage.dependencies) + check.extend(pls) + + return list(map(lambda x: x.name, reversed(build.values()))) + def build(self, store, monitor, libdir): results = {"success": True} diff --git a/test/mod/test_osbuild.py b/test/mod/test_osbuild.py index f767cf63..2e6b4059 100644 --- a/test/mod/test_osbuild.py +++ b/test/mod/test_osbuild.py @@ -17,6 +17,18 @@ from osbuild.pipeline import Manifest, detect_host_runner from .. import test +def names(*lst): + return [x.name for x in lst] + + +class MockStore: + def __init__(self) -> None: + self.have = set() + + def contains(self, pipeline_id): + return pipeline_id in self.have + + class TestDescriptions(unittest.TestCase): @unittest.skipUnless(test.TestBase.can_bind_mount(), "root-only") @@ -108,6 +120,137 @@ class TestDescriptions(unittest.TestCase): check = manifest[i.id] self.assertEqual(i.name, check.name) + # pylint: disable=too-many-statements + def test_on_demand(self): + index = osbuild.meta.Index(os.curdir) + + manifest = Manifest() + noop = index.get_module_info("Stage", "org.osbuild.noop") + noip = index.get_module_info("Input", "org.osbuild.noop") + + # the shared build pipeline + build = manifest.add_pipeline("build", None, None) + build.add_stage(noop, {"option": 1}) + + # a pipeline simulating some intermediate artefact + # that other pipeline need as dependency + dep = manifest.add_pipeline("dep", + "org.osbuild.linux", + build.id) + + dep.add_stage(noop, {"option": 2}) + + # a pipeline that is not linked to the "main" + # assembler artefact and thus should normally + # not be built unless explicitly requested + # has an input that depends on `dep` + ul = manifest.add_pipeline("unlinked", + "org.osbuild.linux", + build.id) + + stage = ul.add_stage(noop, {"option": 3}) + ip = stage.add_input("dep", noip, "org.osbuild.pipeline") + ip.add_reference(dep.id) + + # the main os root file system + rootfs = manifest.add_pipeline("rootfs", + "org.osbuild.inux", + build.id) + stage = rootfs.add_stage(noop, {"option": 4}) + + # the main raw image artefact, depending on "dep" and + # "rootfs" + image = manifest.add_pipeline("image", + "org.osbuild.inux", + build.id) + + stage = image.add_stage(noop, {"option": 5}) + ip = stage.add_input("dep", noip, "org.osbuild.pipeline") + ip.add_reference(dep.id) + + stage = image.add_stage(noop, {"option": 6}) + + # a stage using the rootfs as input (named 'image') + ip = stage.add_input("image", noip, "org.osbuild.pipeline") + ip.add_reference(rootfs.id) + + # some compression of the image, like a qcow2 + qcow2 = manifest.add_pipeline("qcow2", + "org.osbuild.inux", + build.id) + stage = qcow2.add_stage(noop, {"option": 7}) + ip = stage.add_input("image", noip, "org.osbuild.pipeline") + ip.add_reference(image.id) + + fmt = index.get_format_info("osbuild.formats.v2").module + self.assertIsNotNone(fmt) + print(json.dumps(fmt.describe(manifest), indent=2)) + + # The pipeline graph in the manifest with dependencies: + # ├─╼ build + # ├─╼ dep + # │ └ build + # ├─╼ unlinked + # │ ├ build + # │ └ dep + # ├─╼ rootfs + # │ └ build + # ├─╼ image + # │ ├ build + # │ ├ dep + # │ └ rootfs + # └─╼ qcow2 + # ├ build + # └ image + + store = MockStore() + + # check an empty input leads to an empty list + res = manifest.depsolve(store, []) + assert res == [] + + # the build pipeline should resolve to just itself + res = manifest.depsolve(store, names(build)) + assert res == names(build) + + # if we build the 'unlinked' pipeline, we get it + # and its dependencies, dep and build + res = manifest.depsolve(store, names(ul)) + assert res == names(build, dep, ul) + + # building image with nothing in the store should + # result in all pipelines but 'unlinked' + res = manifest.depsolve(store, names(image)) + assert res == names(build, rootfs, dep, image) + + # if we have the 'dep' dependency in the store, + # we should be not be building that + store.have.add(dep.id) + res = manifest.depsolve(store, names(image)) + assert res == names(build, rootfs, image) + + # if we only have the build pipeline in the + # store we should not build that + store.have.clear() + store.have.add(build.id) + res = manifest.depsolve(store, names(image)) + assert res == names(rootfs, dep, image) + + # if we have the final artefact in the store, + # nothing should be built at all + store.have.clear() + store.have.add(image.id) + res = manifest.depsolve(store, names(image)) + assert res == [] + + # we have a checkpoint of the stage in the image + # pipeline with the `dep` dependency, so that + # it effectively only depends on `rootfs` + store.have.clear() + store.have.add(image.stages[0].id) + res = manifest.depsolve(store, names(image)) + assert res == names(build, rootfs, image) + def check_moduleinfo(self, version): index = osbuild.meta.Index(os.curdir)