manifest: implement pipeline depsolving

New function that take a list of pipelines and return the list of
pipelines that need to be build, i.e. the pipelines and all their
dependencies that are not already present in the store.
Add corresponding test.

test/mod/test_osbuild.py

@@ -17,6 +17,18 @@ from osbuild.pipeline import Manifest, detect_host_runner
 from .. import test
 
 
+def names(*lst):
+    return [x.name for x in lst]
+
+
+class MockStore:
+    def __init__(self) -> None:
+        self.have = set()
+
+    def contains(self, pipeline_id):
+        return pipeline_id in self.have
+
+
 class TestDescriptions(unittest.TestCase):
 
     @unittest.skipUnless(test.TestBase.can_bind_mount(), "root-only")
@@ -108,6 +120,137 @@ class TestDescriptions(unittest.TestCase):
             check = manifest[i.id]
             self.assertEqual(i.name, check.name)
 
+    # pylint: disable=too-many-statements
+    def test_on_demand(self):
+        index = osbuild.meta.Index(os.curdir)
+
+        manifest = Manifest()
+        noop = index.get_module_info("Stage", "org.osbuild.noop")
+        noip = index.get_module_info("Input", "org.osbuild.noop")
+
+        # the shared build pipeline
+        build = manifest.add_pipeline("build", None, None)
+        build.add_stage(noop, {"option": 1})
+
+        # a pipeline simulating some intermediate artefact
+        # that other pipeline need as dependency
+        dep = manifest.add_pipeline("dep",
+                                    "org.osbuild.linux",
+                                    build.id)
+
+        dep.add_stage(noop, {"option": 2})
+
+        # a pipeline that is not linked to the "main"
+        # assembler artefact and thus should normally
+        # not be built unless explicitly requested
+        # has an input that depends on `dep`
+        ul = manifest.add_pipeline("unlinked",
+                                   "org.osbuild.linux",
+                                   build.id)
+
+        stage = ul.add_stage(noop, {"option": 3})
+        ip = stage.add_input("dep", noip, "org.osbuild.pipeline")
+        ip.add_reference(dep.id)
+
+        # the main os root file system
+        rootfs = manifest.add_pipeline("rootfs",
+                                       "org.osbuild.inux",
+                                       build.id)
+        stage = rootfs.add_stage(noop, {"option": 4})
+
+        # the main raw image artefact, depending on "dep" and
+        # "rootfs"
+        image = manifest.add_pipeline("image",
+                                      "org.osbuild.inux",
+                                      build.id)
+
+        stage = image.add_stage(noop, {"option": 5})
+        ip = stage.add_input("dep", noip, "org.osbuild.pipeline")
+        ip.add_reference(dep.id)
+
+        stage = image.add_stage(noop, {"option": 6})
+
+        #  a stage using the rootfs as input (named 'image')
+        ip = stage.add_input("image", noip, "org.osbuild.pipeline")
+        ip.add_reference(rootfs.id)
+
+        # some compression of the image, like a qcow2
+        qcow2 = manifest.add_pipeline("qcow2",
+                                      "org.osbuild.inux",
+                                      build.id)
+        stage = qcow2.add_stage(noop, {"option": 7})
+        ip = stage.add_input("image", noip, "org.osbuild.pipeline")
+        ip.add_reference(image.id)
+
+        fmt = index.get_format_info("osbuild.formats.v2").module
+        self.assertIsNotNone(fmt)
+        print(json.dumps(fmt.describe(manifest), indent=2))
+
+        # The pipeline graph in the manifest with dependencies:
+        # ├─╼ build
+        # ├─╼ dep
+        # │ └ build
+        # ├─╼ unlinked
+        # │ ├ build
+        # │ └ dep
+        # ├─╼ rootfs
+        # │ └ build
+        # ├─╼ image
+        # │ ├ build
+        # │ ├ dep
+        # │ └ rootfs
+        # └─╼ qcow2
+        #   ├ build
+        #   └ image
+
+        store = MockStore()
+
+        # check an empty input leads to an empty list
+        res = manifest.depsolve(store, [])
+        assert res == []
+
+        # the build pipeline should resolve to just itself
+        res = manifest.depsolve(store, names(build))
+        assert res == names(build)
+
+        # if we build the 'unlinked' pipeline, we get it
+        # and its dependencies, dep and build
+        res = manifest.depsolve(store, names(ul))
+        assert res == names(build, dep, ul)
+
+        # building image with nothing in the store should
+        # result in all pipelines but 'unlinked'
+        res = manifest.depsolve(store, names(image))
+        assert res == names(build, rootfs, dep, image)
+
+        # if we have the 'dep' dependency in the store,
+        # we should be not be building that
+        store.have.add(dep.id)
+        res = manifest.depsolve(store, names(image))
+        assert res == names(build, rootfs, image)
+
+        # if we only have the build pipeline in the
+        # store we should not build that
+        store.have.clear()
+        store.have.add(build.id)
+        res = manifest.depsolve(store, names(image))
+        assert res == names(rootfs, dep, image)
+
+        # if we have the final artefact in the store,
+        # nothing should be built at all
+        store.have.clear()
+        store.have.add(image.id)
+        res = manifest.depsolve(store, names(image))
+        assert res == []
+
+        # we have a checkpoint of the stage in the image
+        # pipeline with the `dep` dependency, so that
+        # it effectively only depends on `rootfs`
+        store.have.clear()
+        store.have.add(image.stages[0].id)
+        res = manifest.depsolve(store, names(image))
+        assert res == names(build, rootfs, image)
+
     def check_moduleinfo(self, version):
         index = osbuild.meta.Index(os.curdir)