diff --git a/devices/org.osbuild.loopback b/devices/org.osbuild.loopback index 955a91ff..fd0cf67e 100755 --- a/devices/org.osbuild.loopback +++ b/devices/org.osbuild.loopback @@ -78,7 +78,7 @@ class LoopbackService(devices.DeviceService): return lo - def open(self, devpath: str, tree: str, options: Dict): + def open(self, devpath: str, parent: str, tree: str, options: Dict): filename = options["filename"] self.sector_size = options.get("sector-size", 512) start = options.get("start", 0) * self.sector_size diff --git a/osbuild/devices.py b/osbuild/devices.py index 7546c73e..1e40160f 100644 --- a/osbuild/devices.py +++ b/osbuild/devices.py @@ -26,9 +26,10 @@ class Device: A single device with its corresponding options """ - def __init__(self, name, info, options: Dict): + def __init__(self, name, info, parent, options: Dict): self.name = name self.info = info + self.parent = parent self.options = options or {} self.id = self.calc_id() @@ -36,16 +37,21 @@ class Device: # NB: Since the name of the device is arbitrary or prescribed # by the stage, it is not included in the id calculation. m = hashlib.sha256() + m.update(json.dumps(self.info.name, sort_keys=True).encode()) + if self.parent: + m.update(json.dumps(self.parent.id, sort_keys=True).encode()) m.update(json.dumps(self.options, sort_keys=True).encode()) return m.hexdigest() - def open(self, mgr: host.ServiceManager, dev: str, tree: str) -> Dict: + def open(self, mgr: host.ServiceManager, dev: str, parent: str, tree: str) -> Dict: args = { # global options "dev": dev, "tree": tree, + "parent": parent, + # per device options "options": self.options, } @@ -60,7 +66,7 @@ class DeviceService(host.Service): """Device host service""" @abc.abstractmethod - def open(self, devpath: str, tree: str, options: Dict): + def open(self, devpath: str, parent: str, tree: str, options: Dict): """Open a specific device This method must be implemented by the specific device service. @@ -78,7 +84,10 @@ class DeviceService(host.Service): def dispatch(self, method: str, args, _fds): if method == "open": - r = self.open(args["dev"], args["tree"], args["options"]) + r = self.open(args["dev"], + args["parent"], + args["tree"], + args["options"]) return r, None if method == "close": r = self.close() diff --git a/osbuild/formats/v2.py b/osbuild/formats/v2.py index 0ea65330..4b5190e7 100644 --- a/osbuild/formats/v2.py +++ b/osbuild/formats/v2.py @@ -175,14 +175,64 @@ def resolve_ref(name: str, manifest: Manifest) -> str: return target.id +def sort_devices(devices: Dict) -> Dict: + """Sort the devices so that dependencies are in the correct order + + We need to ensure that parents are sorted before the devices that + depend on them. For this we keep a list of devices that need to + be processed and iterate over that list as long as it has devices + in them and we make progress, i.e. the length changes. + """ + result = {} + todo = list(devices.keys()) + + while todo: + before = len(todo) + + for i, name in enumerate(todo): + desc = devices[name] + + parent = desc.get("parent") + if parent and not parent in result: + # if the parent is not in the `result` list, it must + # be in `todo`; otherwise it is missing + if parent not in todo: + msg = f"Missing parent device '{parent}' for '{name}'" + raise ValueError(msg) + + continue + + # no parent, or parent already present, ok to add to the + # result and "remove" from the todo list, by setting the + # contents to `None`. + result[name] = desc + todo[i] = None + + todo = list(filter(bool, todo)) + if len(todo) == before: + # we made no progress, which means that all devices in todo + # depend on other devices in todo, hence we have a cycle + raise ValueError("Cycle detected in 'devices'") + + return result + + def load_device(name: str, description: Dict, index: Index, stage: Stage): device_type = description["type"] options = description.get("options", {}) + parent = description.get("parent") + + if parent: + device = stage.devices.get(parent) + if not parent: + raise ValueError(f"Unknown parent device: {parent}") + parent = device info = index.get_module_info("Device", device_type) + if not info: raise TypeError(f"Missing meta information for {device_type}") - stage.add_device(name, info, options) + stage.add_device(name, info, parent, options) def load_input(name: str, description: Dict, index: Index, stage: Stage, manifest: Manifest, source_refs: set): @@ -244,6 +294,8 @@ def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Ma stage = pipeline.add_stage(info, opts) devs = description.get("devices", {}) + devs = sort_devices(devs) + for name, desc in devs.items(): load_device(name, desc, index, stage) diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index c0ffd5a9..87354373 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -69,8 +69,8 @@ class Stage: self.inputs[name] = ip return ip - def add_device(self, name, info, options): - dev = Device(name, info, options) + def add_device(self, name, info, parent, options): + dev = Device(name, info, parent, options) self.devices[name] = dev return dev @@ -161,7 +161,10 @@ class Stage: inputs[key] = data for key, dev in self.devices.items(): - reply = dev.open(mgr, build_root.dev, tree) + parent = None + if dev.parent: + parent = devices[dev.parent.name]["path"] + reply = dev.open(mgr, build_root.dev, parent, tree) devices[key] = reply for key, mount in self.mounts.items(): diff --git a/schemas/osbuild2.json b/schemas/osbuild2.json index 5261b751..4d66c1bc 100644 --- a/schemas/osbuild2.json +++ b/schemas/osbuild2.json @@ -29,6 +29,7 @@ "required": ["type"], "properties": { "type": { "type": "string" }, + "parent": { "type": "string" }, "options": { "type": "object", "additionalProperties": true diff --git a/test/mod/test_fmt_v2.py b/test/mod/test_fmt_v2.py index 047c5aef..1f4f3b73 100644 --- a/test/mod/test_fmt_v2.py +++ b/test/mod/test_fmt_v2.py @@ -3,6 +3,7 @@ # import copy +import itertools import os import unittest @@ -306,3 +307,54 @@ class TestFormatV2(unittest.TestCase): with self.assertRaises(ValueError): self.load_manifest(pipeline) + + def test_device_sorting(self): + fmt = self.index.get_format_info("osbuild.formats.v2").module + assert(fmt) + + self_cycle = { + "a": {"parent": "a"}, + } + + with self.assertRaises(ValueError): + fmt.sort_devices(self_cycle) + + cycle = { + "a": {"parent": "b"}, + "b": {"parent": "a"}, + } + + with self.assertRaises(ValueError): + fmt.sort_devices(cycle) + + missing_parent = { + "a": {"parent": "b"}, + "b": {"parent": "c"}, + } + + with self.assertRaises(ValueError): + fmt.sort_devices(missing_parent) + + def ensure_sorted(devices): + check = {} + + for name, dev in devices.items(): + + parent = dev.get("parent") + if parent: + assert parent in check + + check[name] = dev + + assert devices == check + + devices = { + "a": {"parent": "d"}, + "b": {"parent": "a"}, + "c": {"parent": None}, + "d": {"parent": "c"}, + } + + for check in itertools.permutations(devices.keys()): + before = {name: devices[name] for name in check} + ensure_sorted(fmt.sort_devices(before)) diff --git a/test/run/test_devices.py b/test/run/test_devices.py index 94034b3c..80b25c89 100755 --- a/test/run/test_devices.py +++ b/test/run/test_devices.py @@ -45,10 +45,10 @@ def test_loopback_basic(tmpdir): "size": size // 512 # size is in sectors / blocks } - dev = devices.Device("loop", info, options) + dev = devices.Device("loop", info, None, options) with host.ServiceManager() as mgr: - reply = dev.open(mgr, devpath, tree) + reply = dev.open(mgr, devpath, None, tree) assert reply assert reply["path"]