From 749912c75aea4f565bfdf220c2d6c83466fb3fd2 Mon Sep 17 00:00:00 2001 From: Christian Kellner Date: Mon, 25 Oct 2021 16:06:12 +0200 Subject: [PATCH] manifest: implement pipeline depsolving New function that take a list of pipelines and return the list of pipelines that need to be build, i.e. the pipelines and all their dependencies that are not already present in the store. Add corresponding test. --- osbuild/pipeline.py | 63 ++++++++++++++++- test/mod/test_osbuild.py | 143 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index b52d7cd4..abe5398f 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -3,7 +3,7 @@ import contextlib import hashlib import json import os -from typing import Dict, Iterator, List, Optional +from typing import Dict, Generator, Iterable, Iterator, List, Optional from .api import API from . import buildroot @@ -64,6 +64,18 @@ class Stage: m.update(json.dumps(data, sort_keys=True).encode()) return m.hexdigest() + @property + def dependencies(self) -> Generator[str, None, None]: + """Return a list of pipeline ids this stage depends on""" + + for ip in self.inputs.values(): + + if ip.origin != "org.osbuild.pipeline": + continue + + for ref in ip.refs: + yield ref + def add_input(self, name, info, origin, options=None): ip = Input(name, info, origin, options or {}) self.inputs[name] = ip @@ -344,6 +356,55 @@ class Manifest: for source in self.sources: source.download(mgr, store, libdir) + def depsolve(self, store, targets: Iterable[str]) -> List[str]: + """Return the list of pipelines that need to be built + + Given a list of target pipelines, return the names + of all pipelines and their dependencies that are not + already present in the store. + """ + + # A stack of pipelines to check if they need to be built + check = list(map(self.get, targets)) + + # The ordered result "set", will be reversed at the end + build = collections.OrderedDict() + + while check: + pl = check.pop() # get the last(!) item + + if store.contains(pl.id): + continue + + # The store does not have this pipeline, it needs to + # be built, add it to the ordered result set and + # ensure it is at the end, i.e. built before previously + # checked items. NB: the result set is reversed before + # it gets returned. This ensures that a dependency that + # gets checked multiple times, like a build pipeline, + # always gets built before its dependent pipeline. + build[pl.id] = pl + build.move_to_end(pl.id) + + # Add all dependencies to the stack of things to check, + # starting with the build pipeline, if there is one + if pl.build: + check.append(self.get(pl.build)) + + # Stages depend on other pipeline via pipeline inputs. + # We check in reversed order until we hit a checkpoint + for stage in reversed(pl.stages): + + # we stop if we have a checkpoint, i.e. we don't + # need to build any stages after that checkpoint + if store.contains(stage.id): + break + + pls = map(self.get, stage.dependencies) + check.extend(pls) + + return list(map(lambda x: x.name, reversed(build.values()))) + def build(self, store, monitor, libdir): results = {"success": True} diff --git a/test/mod/test_osbuild.py b/test/mod/test_osbuild.py index f767cf63..2e6b4059 100644 --- a/test/mod/test_osbuild.py +++ b/test/mod/test_osbuild.py @@ -17,6 +17,18 @@ from osbuild.pipeline import Manifest, detect_host_runner from .. import test +def names(*lst): + return [x.name for x in lst] + + +class MockStore: + def __init__(self) -> None: + self.have = set() + + def contains(self, pipeline_id): + return pipeline_id in self.have + + class TestDescriptions(unittest.TestCase): @unittest.skipUnless(test.TestBase.can_bind_mount(), "root-only") @@ -108,6 +120,137 @@ class TestDescriptions(unittest.TestCase): check = manifest[i.id] self.assertEqual(i.name, check.name) + # pylint: disable=too-many-statements + def test_on_demand(self): + index = osbuild.meta.Index(os.curdir) + + manifest = Manifest() + noop = index.get_module_info("Stage", "org.osbuild.noop") + noip = index.get_module_info("Input", "org.osbuild.noop") + + # the shared build pipeline + build = manifest.add_pipeline("build", None, None) + build.add_stage(noop, {"option": 1}) + + # a pipeline simulating some intermediate artefact + # that other pipeline need as dependency + dep = manifest.add_pipeline("dep", + "org.osbuild.linux", + build.id) + + dep.add_stage(noop, {"option": 2}) + + # a pipeline that is not linked to the "main" + # assembler artefact and thus should normally + # not be built unless explicitly requested + # has an input that depends on `dep` + ul = manifest.add_pipeline("unlinked", + "org.osbuild.linux", + build.id) + + stage = ul.add_stage(noop, {"option": 3}) + ip = stage.add_input("dep", noip, "org.osbuild.pipeline") + ip.add_reference(dep.id) + + # the main os root file system + rootfs = manifest.add_pipeline("rootfs", + "org.osbuild.inux", + build.id) + stage = rootfs.add_stage(noop, {"option": 4}) + + # the main raw image artefact, depending on "dep" and + # "rootfs" + image = manifest.add_pipeline("image", + "org.osbuild.inux", + build.id) + + stage = image.add_stage(noop, {"option": 5}) + ip = stage.add_input("dep", noip, "org.osbuild.pipeline") + ip.add_reference(dep.id) + + stage = image.add_stage(noop, {"option": 6}) + + # a stage using the rootfs as input (named 'image') + ip = stage.add_input("image", noip, "org.osbuild.pipeline") + ip.add_reference(rootfs.id) + + # some compression of the image, like a qcow2 + qcow2 = manifest.add_pipeline("qcow2", + "org.osbuild.inux", + build.id) + stage = qcow2.add_stage(noop, {"option": 7}) + ip = stage.add_input("image", noip, "org.osbuild.pipeline") + ip.add_reference(image.id) + + fmt = index.get_format_info("osbuild.formats.v2").module + self.assertIsNotNone(fmt) + print(json.dumps(fmt.describe(manifest), indent=2)) + + # The pipeline graph in the manifest with dependencies: + # ├─╼ build + # ├─╼ dep + # │ └ build + # ├─╼ unlinked + # │ ├ build + # │ └ dep + # ├─╼ rootfs + # │ └ build + # ├─╼ image + # │ ├ build + # │ ├ dep + # │ └ rootfs + # └─╼ qcow2 + # ├ build + # └ image + + store = MockStore() + + # check an empty input leads to an empty list + res = manifest.depsolve(store, []) + assert res == [] + + # the build pipeline should resolve to just itself + res = manifest.depsolve(store, names(build)) + assert res == names(build) + + # if we build the 'unlinked' pipeline, we get it + # and its dependencies, dep and build + res = manifest.depsolve(store, names(ul)) + assert res == names(build, dep, ul) + + # building image with nothing in the store should + # result in all pipelines but 'unlinked' + res = manifest.depsolve(store, names(image)) + assert res == names(build, rootfs, dep, image) + + # if we have the 'dep' dependency in the store, + # we should be not be building that + store.have.add(dep.id) + res = manifest.depsolve(store, names(image)) + assert res == names(build, rootfs, image) + + # if we only have the build pipeline in the + # store we should not build that + store.have.clear() + store.have.add(build.id) + res = manifest.depsolve(store, names(image)) + assert res == names(rootfs, dep, image) + + # if we have the final artefact in the store, + # nothing should be built at all + store.have.clear() + store.have.add(image.id) + res = manifest.depsolve(store, names(image)) + assert res == [] + + # we have a checkpoint of the stage in the image + # pipeline with the `dep` dependency, so that + # it effectively only depends on `rootfs` + store.have.clear() + store.have.add(image.stages[0].id) + res = manifest.depsolve(store, names(image)) + assert res == names(build, rootfs, image) + def check_moduleinfo(self, version): index = osbuild.meta.Index(os.curdir)