device: add support for parent devices

This allows device nesting, i.e. one device being opened inside another
one.

devices/org.osbuild.loopback

@@ -78,7 +78,7 @@ class LoopbackService(devices.DeviceService):
 
         return lo
 
-    def open(self, devpath: str, tree: str, options: Dict):
+    def open(self, devpath: str, parent: str, tree: str, options: Dict):
         filename = options["filename"]
         self.sector_size = options.get("sector-size", 512)
         start = options.get("start", 0) * self.sector_size

osbuild/devices.py

@@ -26,9 +26,10 @@ class Device:
     A single device with its corresponding options
     """
 
-    def __init__(self, name, info, options: Dict):
+    def __init__(self, name, info, parent, options: Dict):
         self.name = name
         self.info = info
+        self.parent = parent
         self.options = options or {}
         self.id = self.calc_id()
 
@@ -36,16 +37,21 @@ class Device:
         # NB: Since the name of the device is arbitrary or prescribed
         # by the stage, it is not included in the id calculation.
         m = hashlib.sha256()
+
         m.update(json.dumps(self.info.name, sort_keys=True).encode())
+        if self.parent:
+            m.update(json.dumps(self.parent.id, sort_keys=True).encode())
         m.update(json.dumps(self.options, sort_keys=True).encode())
         return m.hexdigest()
 
-    def open(self, mgr: host.ServiceManager, dev: str, tree: str) -> Dict:
+    def open(self, mgr: host.ServiceManager, dev: str, parent: str, tree: str) -> Dict:
         args = {
             # global options
             "dev": dev,
             "tree": tree,
 
+            "parent": parent,
+
             # per device options
             "options": self.options,
         }
@@ -60,7 +66,7 @@ class DeviceService(host.Service):
     """Device host service"""
 
     @abc.abstractmethod
-    def open(self, devpath: str, tree: str, options: Dict):
+    def open(self, devpath: str, parent: str, tree: str, options: Dict):
         """Open a specific device
 
         This method must be implemented by the specific device service.
@@ -78,7 +84,10 @@ class DeviceService(host.Service):
 
     def dispatch(self, method: str, args, _fds):
         if method == "open":
-            r = self.open(args["dev"], args["tree"], args["options"])
+            r = self.open(args["dev"],
+                          args["parent"],
+                          args["tree"],
+                          args["options"])
             return r, None
         if method == "close":
             r = self.close()

osbuild/formats/v2.py

@@ -175,14 +175,64 @@ def resolve_ref(name: str, manifest: Manifest) -> str:
     return target.id
 
 
+def sort_devices(devices: Dict) -> Dict:
+    """Sort the devices so that dependencies are in the correct order
+
+    We need to ensure that parents are sorted before the devices that
+    depend on them. For this we keep a list of devices that need to
+    be processed and iterate over that list as long as it has devices
+    in them and we make progress, i.e. the length changes.
+    """
+    result = {}
+    todo = list(devices.keys())
+
+    while todo:
+        before = len(todo)
+
+        for i, name in enumerate(todo):
+            desc = devices[name]
+
+            parent = desc.get("parent")
+            if parent and not parent in result:
+                # if the parent is not in the `result` list, it must
+                # be in `todo`; otherwise it is missing
+                if parent not in todo:
+                    msg = f"Missing parent device '{parent}' for '{name}'"
+                    raise ValueError(msg)
+
+                continue
+
+            # no parent, or parent already present, ok to add to the
+            # result and "remove" from the todo list, by setting the
+            # contents to `None`.
+            result[name] = desc
+            todo[i] = None
+
+        todo = list(filter(bool, todo))
+        if len(todo) == before:
+            # we made no progress, which means that all devices in todo
+            # depend on other devices in todo, hence we have a cycle
+            raise ValueError("Cycle detected in 'devices'")
+
+    return result
+
+
 def load_device(name: str, description: Dict, index: Index, stage: Stage):
     device_type = description["type"]
     options = description.get("options", {})
+    parent = description.get("parent")
+
+    if parent:
+        device = stage.devices.get(parent)
+        if not parent:
+            raise ValueError(f"Unknown parent device: {parent}")
+        parent = device
 
     info = index.get_module_info("Device", device_type)
+
     if not info:
         raise TypeError(f"Missing meta information for {device_type}")
-    stage.add_device(name, info, options)
+    stage.add_device(name, info, parent, options)
 
 
 def load_input(name: str, description: Dict, index: Index, stage: Stage, manifest: Manifest, source_refs: set):
@@ -244,6 +294,8 @@ def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Ma
     stage = pipeline.add_stage(info, opts)
 
     devs = description.get("devices", {})
+    devs = sort_devices(devs)
+
     for name, desc in devs.items():
         load_device(name, desc, index, stage)
 

osbuild/pipeline.py

@@ -69,8 +69,8 @@ class Stage:
         self.inputs[name] = ip
         return ip
 
-    def add_device(self, name, info, options):
-        dev = Device(name, info, options)
+    def add_device(self, name, info, parent, options):
+        dev = Device(name, info, parent, options)
         self.devices[name] = dev
         return dev
 
@@ -161,7 +161,10 @@ class Stage:
                 inputs[key] = data
 
             for key, dev in self.devices.items():
-                reply = dev.open(mgr, build_root.dev, tree)
+                parent = None
+                if dev.parent:
+                    parent = devices[dev.parent.name]["path"]
+                reply = dev.open(mgr, build_root.dev, parent, tree)
                 devices[key] = reply
 
             for key, mount in self.mounts.items():

schemas/osbuild2.json

@@ -29,6 +29,7 @@
       "required": ["type"],
       "properties": {
         "type": { "type": "string" },
+        "parent": { "type": "string" },
         "options": {
           "type": "object",
           "additionalProperties": true

test/mod/test_fmt_v2.py

@@ -3,6 +3,7 @@
 #
 
 import copy
+import itertools
 import os
 import unittest
 
@@ -306,3 +307,54 @@ class TestFormatV2(unittest.TestCase):
 
         with self.assertRaises(ValueError):
             self.load_manifest(pipeline)
+
+    def test_device_sorting(self):
+        fmt = self.index.get_format_info("osbuild.formats.v2").module
+        assert(fmt)
+
+        self_cycle = {
+            "a": {"parent": "a"},
+        }
+
+        with self.assertRaises(ValueError):
+            fmt.sort_devices(self_cycle)
+
+        cycle = {
+            "a": {"parent": "b"},
+            "b": {"parent": "a"},
+        }
+
+        with self.assertRaises(ValueError):
+            fmt.sort_devices(cycle)
+
+        missing_parent = {
+            "a": {"parent": "b"},
+            "b": {"parent": "c"},
+        }
+
+        with self.assertRaises(ValueError):
+            fmt.sort_devices(missing_parent)
+
+        def ensure_sorted(devices):
+            check = {}
+
+            for name, dev in devices.items():
+
+                parent = dev.get("parent")
+                if parent:
+                    assert parent in check
+
+                check[name] = dev
+
+            assert devices == check
+
+        devices = {
+            "a": {"parent": "d"},
+            "b": {"parent": "a"},
+            "c": {"parent": None},
+            "d": {"parent": "c"},
+        }
+
+        for check in itertools.permutations(devices.keys()):
+            before = {name: devices[name] for name in check}
+            ensure_sorted(fmt.sort_devices(before))

test/run/test_devices.py

@@ -45,10 +45,10 @@ def test_loopback_basic(tmpdir):
         "size": size // 512  # size is in sectors / blocks
     }
 
-    dev = devices.Device("loop", info, options)
+    dev = devices.Device("loop", info, None, options)
 
     with host.ServiceManager() as mgr:
-        reply = dev.open(mgr, devpath, tree)
+        reply = dev.open(mgr, devpath, None, tree)
         assert reply
         assert reply["path"]