manifest: implement pipeline depsolving
New function that take a list of pipelines and return the list of pipelines that need to be build, i.e. the pipelines and all their dependencies that are not already present in the store. Add corresponding test.
This commit is contained in:
Christian Kellner
Tom Gundersen
parent
a2404c9ec9
commit
749912c75a
2 changed files with 205 additions and 1 deletions
|
|
@ -3,7 +3,7 @@ import contextlib
|
|||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import Dict, Iterator, List, Optional
|
||||
from typing import Dict, Generator, Iterable, Iterator, List, Optional
|
||||
|
||||
from .api import API
|
||||
from . import buildroot
|
||||
|
|
@ -64,6 +64,18 @@ class Stage:
|
|||
m.update(json.dumps(data, sort_keys=True).encode())
|
||||
return m.hexdigest()
|
||||
|
||||
@property
|
||||
def dependencies(self) -> Generator[str, None, None]:
|
||||
"""Return a list of pipeline ids this stage depends on"""
|
||||
|
||||
for ip in self.inputs.values():
|
||||
|
||||
if ip.origin != "org.osbuild.pipeline":
|
||||
continue
|
||||
|
||||
for ref in ip.refs:
|
||||
yield ref
|
||||
|
||||
def add_input(self, name, info, origin, options=None):
|
||||
ip = Input(name, info, origin, options or {})
|
||||
self.inputs[name] = ip
|
||||
|
|
@ -344,6 +356,55 @@ class Manifest:
|
|||
for source in self.sources:
|
||||
source.download(mgr, store, libdir)
|
||||
|
||||
def depsolve(self, store, targets: Iterable[str]) -> List[str]:
|
||||
"""Return the list of pipelines that need to be built
|
||||
|
||||
Given a list of target pipelines, return the names
|
||||
of all pipelines and their dependencies that are not
|
||||
already present in the store.
|
||||
"""
|
||||
|
||||
# A stack of pipelines to check if they need to be built
|
||||
check = list(map(self.get, targets))
|
||||
|
||||
# The ordered result "set", will be reversed at the end
|
||||
build = collections.OrderedDict()
|
||||
|
||||
while check:
|
||||
pl = check.pop() # get the last(!) item
|
||||
|
||||
if store.contains(pl.id):
|
||||
continue
|
||||
|
||||
# The store does not have this pipeline, it needs to
|
||||
# be built, add it to the ordered result set and
|
||||
# ensure it is at the end, i.e. built before previously
|
||||
# checked items. NB: the result set is reversed before
|
||||
# it gets returned. This ensures that a dependency that
|
||||
# gets checked multiple times, like a build pipeline,
|
||||
# always gets built before its dependent pipeline.
|
||||
build[pl.id] = pl
|
||||
build.move_to_end(pl.id)
|
||||
|
||||
# Add all dependencies to the stack of things to check,
|
||||
# starting with the build pipeline, if there is one
|
||||
if pl.build:
|
||||
check.append(self.get(pl.build))
|
||||
|
||||
# Stages depend on other pipeline via pipeline inputs.
|
||||
# We check in reversed order until we hit a checkpoint
|
||||
for stage in reversed(pl.stages):
|
||||
|
||||
# we stop if we have a checkpoint, i.e. we don't
|
||||
# need to build any stages after that checkpoint
|
||||
if store.contains(stage.id):
|
||||
break
|
||||
|
||||
pls = map(self.get, stage.dependencies)
|
||||
check.extend(pls)
|
||||
|
||||
return list(map(lambda x: x.name, reversed(build.values())))
|
||||
|
||||
def build(self, store, monitor, libdir):
|
||||
results = {"success": True}
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,18 @@ from osbuild.pipeline import Manifest, detect_host_runner
|
|||
from .. import test
|
||||
|
||||
|
||||
def names(*lst):
|
||||
return [x.name for x in lst]
|
||||
|
||||
|
||||
class MockStore:
|
||||
def __init__(self) -> None:
|
||||
self.have = set()
|
||||
|
||||
def contains(self, pipeline_id):
|
||||
return pipeline_id in self.have
|
||||
|
||||
|
||||
class TestDescriptions(unittest.TestCase):
|
||||
|
||||
@unittest.skipUnless(test.TestBase.can_bind_mount(), "root-only")
|
||||
|
|
@ -108,6 +120,137 @@ class TestDescriptions(unittest.TestCase):
|
|||
check = manifest[i.id]
|
||||
self.assertEqual(i.name, check.name)
|
||||
|
||||
# pylint: disable=too-many-statements
|
||||
def test_on_demand(self):
|
||||
index = osbuild.meta.Index(os.curdir)
|
||||
|
||||
manifest = Manifest()
|
||||
noop = index.get_module_info("Stage", "org.osbuild.noop")
|
||||
noip = index.get_module_info("Input", "org.osbuild.noop")
|
||||
|
||||
# the shared build pipeline
|
||||
build = manifest.add_pipeline("build", None, None)
|
||||
build.add_stage(noop, {"option": 1})
|
||||
|
||||
# a pipeline simulating some intermediate artefact
|
||||
# that other pipeline need as dependency
|
||||
dep = manifest.add_pipeline("dep",
|
||||
"org.osbuild.linux",
|
||||
build.id)
|
||||
|
||||
dep.add_stage(noop, {"option": 2})
|
||||
|
||||
# a pipeline that is not linked to the "main"
|
||||
# assembler artefact and thus should normally
|
||||
# not be built unless explicitly requested
|
||||
# has an input that depends on `dep`
|
||||
ul = manifest.add_pipeline("unlinked",
|
||||
"org.osbuild.linux",
|
||||
build.id)
|
||||
|
||||
stage = ul.add_stage(noop, {"option": 3})
|
||||
ip = stage.add_input("dep", noip, "org.osbuild.pipeline")
|
||||
ip.add_reference(dep.id)
|
||||
|
||||
# the main os root file system
|
||||
rootfs = manifest.add_pipeline("rootfs",
|
||||
"org.osbuild.inux",
|
||||
build.id)
|
||||
stage = rootfs.add_stage(noop, {"option": 4})
|
||||
|
||||
# the main raw image artefact, depending on "dep" and
|
||||
# "rootfs"
|
||||
image = manifest.add_pipeline("image",
|
||||
"org.osbuild.inux",
|
||||
build.id)
|
||||
|
||||
stage = image.add_stage(noop, {"option": 5})
|
||||
ip = stage.add_input("dep", noip, "org.osbuild.pipeline")
|
||||
ip.add_reference(dep.id)
|
||||
|
||||
stage = image.add_stage(noop, {"option": 6})
|
||||
|
||||
# a stage using the rootfs as input (named 'image')
|
||||
ip = stage.add_input("image", noip, "org.osbuild.pipeline")
|
||||
ip.add_reference(rootfs.id)
|
||||
|
||||
# some compression of the image, like a qcow2
|
||||
qcow2 = manifest.add_pipeline("qcow2",
|
||||
"org.osbuild.inux",
|
||||
build.id)
|
||||
stage = qcow2.add_stage(noop, {"option": 7})
|
||||
ip = stage.add_input("image", noip, "org.osbuild.pipeline")
|
||||
ip.add_reference(image.id)
|
||||
|
||||
fmt = index.get_format_info("osbuild.formats.v2").module
|
||||
self.assertIsNotNone(fmt)
|
||||
print(json.dumps(fmt.describe(manifest), indent=2))
|
||||
|
||||
# The pipeline graph in the manifest with dependencies:
|
||||
# ├─╼ build
|
||||
# ├─╼ dep
|
||||
# │ └ build
|
||||
# ├─╼ unlinked
|
||||
# │ ├ build
|
||||
# │ └ dep
|
||||
# ├─╼ rootfs
|
||||
# │ └ build
|
||||
# ├─╼ image
|
||||
# │ ├ build
|
||||
# │ ├ dep
|
||||
# │ └ rootfs
|
||||
# └─╼ qcow2
|
||||
# ├ build
|
||||
# └ image
|
||||
|
||||
store = MockStore()
|
||||
|
||||
# check an empty input leads to an empty list
|
||||
res = manifest.depsolve(store, [])
|
||||
assert res == []
|
||||
|
||||
# the build pipeline should resolve to just itself
|
||||
res = manifest.depsolve(store, names(build))
|
||||
assert res == names(build)
|
||||
|
||||
# if we build the 'unlinked' pipeline, we get it
|
||||
# and its dependencies, dep and build
|
||||
res = manifest.depsolve(store, names(ul))
|
||||
assert res == names(build, dep, ul)
|
||||
|
||||
# building image with nothing in the store should
|
||||
# result in all pipelines but 'unlinked'
|
||||
res = manifest.depsolve(store, names(image))
|
||||
assert res == names(build, rootfs, dep, image)
|
||||
|
||||
# if we have the 'dep' dependency in the store,
|
||||
# we should be not be building that
|
||||
store.have.add(dep.id)
|
||||
res = manifest.depsolve(store, names(image))
|
||||
assert res == names(build, rootfs, image)
|
||||
|
||||
# if we only have the build pipeline in the
|
||||
# store we should not build that
|
||||
store.have.clear()
|
||||
store.have.add(build.id)
|
||||
res = manifest.depsolve(store, names(image))
|
||||
assert res == names(rootfs, dep, image)
|
||||
|
||||
# if we have the final artefact in the store,
|
||||
# nothing should be built at all
|
||||
store.have.clear()
|
||||
store.have.add(image.id)
|
||||
res = manifest.depsolve(store, names(image))
|
||||
assert res == []
|
||||
|
||||
# we have a checkpoint of the stage in the image
|
||||
# pipeline with the `dep` dependency, so that
|
||||
# it effectively only depends on `rootfs`
|
||||
store.have.clear()
|
||||
store.have.add(image.stages[0].id)
|
||||
res = manifest.depsolve(store, names(image))
|
||||
assert res == names(build, rootfs, image)
|
||||
|
||||
def check_moduleinfo(self, version):
|
||||
index = osbuild.meta.Index(os.curdir)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue