osbuild: introduce device host service

A new host service that provides device functionality to stages.
Since stages run in a container and are restricted from creating
device nodes, all device handling is done in the main osbuild
process. Currently this is done with the help of APIs and RPC,
e.g. `LoopServer`. Device host services on the other hand allow
declaring devices in the manifest itself and then osbuild will
prepare all devices before running the stage. One desired effect
is that it makes device handling transparent to the stages, e.g.
they don't have to know about loopback devices, LVM or LUKS.
Another result is that specific device handling is now modular
like Inputs and Source are and thus moved out of osbuild itself.

osbuild/devices.py

@@ -0,0 +1,84 @@
+"""
+Device Handling for pipeline stages
+
+Specific type of artifacts require device support, such as
+loopback devices or device mapper. Since stages are always
+run in a container and are isolated from the host, they do
+not have direct access to devices and specifically can not
+setup new ones.
+Therefore device handling is done at the osbuild level with
+the help of a device host services. Device specific modules
+provide the actual functionality and thus the core device
+support in osbuild itself is abstract.
+"""
+
+import abc
+import hashlib
+import json
+
+from typing import Dict
+
+from osbuild import host
+
+
+class Device:
+    """
+    A single device with its corresponding options
+    """
+
+    def __init__(self, name, info, options: Dict):
+        self.name = name
+        self.info = info
+        self.options = options or {}
+        self.id = self.calc_id()
+
+    def calc_id(self):
+        # NB: Since the name of the device is arbitrary or prescribed
+        # by the stage, it is not included in the id calculation.
+        m = hashlib.sha256()
+        m.update(json.dumps(self.info.name, sort_keys=True).encode())
+        m.update(json.dumps(self.options, sort_keys=True).encode())
+        return m.hexdigest()
+
+    def open(self, mgr: host.ServiceManager, dev: str, tree: str) -> Dict:
+        args = {
+            # global options
+            "dev": dev,
+            "tree": tree,
+
+            # per device options
+            "options": self.options,
+        }
+
+        client = mgr.start(f"device/{self.name}", self.info.path)
+        res = client.call("open", args)
+
+        return res
+
+
+class DeviceService(host.Service):
+    """Device host service"""
+
+    @abc.abstractmethod
+    def open(self, devpath: str, tree: str, options: Dict):
+        """Open a specific device
+
+        This method must be implemented by the specific device service.
+        It should open the device and create a device node in `devpath`.
+        The return value must contain the relative path to the device
+        node.
+        """
+
+    @abc.abstractmethod
+    def close(self):
+        """Close the device"""
+
+    def stop(self):
+        self.close()
+
+    def dispatch(self, method: str, args, _fds):
+        if method == "open":
+            r = self.open(args["dev"], args["tree"], args["options"])
+            return r, None
+
+        raise host.ProtocolError("Unknown method")

osbuild/formats/v2.py

@@ -28,6 +28,23 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
         pl = manifest[pid]
         return f"name:{pl.name}"
 
+    def describe_device(dev):
+        desc = {
+            "type": dev.info.name
+        }
+
+        if dev.options:
+            desc["options"] = dev.options
+
+        return desc
+
+    def describe_devices(devs: Dict):
+        desc = {
+            name: describe_device(dev)
+            for name, dev in devs.items()
+        }
+        return desc
+
     def describe_input(ip: Input):
         origin = ip.origin
         desc = {
@@ -66,6 +83,10 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
         if s.options:
             desc["options"] = s.options
 
+        devs = describe_devices(s.devices)
+        if devs:
+            desc["devices"] = devs
+
         ips = describe_inputs(s.inputs)
         if ips:
             desc["inputs"] = ips
@@ -130,6 +151,16 @@ def resolve_ref(name: str, manifest: Manifest) -> str:
     return target.id
 
 
+def load_device(name: str, description: Dict, index: Index, stage: Stage):
+    device_type = description["type"]
+    options = description.get("options", {})
+
+    info = index.get_module_info("Device", device_type)
+    if not info:
+        raise TypeError(f"Missing meta information for {device_type}")
+    stage.add_device(name, info, options)
+
+
 def load_input(name: str, description: Dict, index: Index, stage: Stage, manifest: Manifest):
     input_type = description["type"]
     origin = description["origin"]
@@ -163,6 +194,10 @@ def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Ma
 
     stage = pipeline.add_stage(info, opts)
 
+    devs = description.get("devices", {})
+    for name, desc in devs.items():
+        load_device(name, desc, index, stage)
+
     ips = description.get("inputs", {})
     for name, desc in ips.items():
         load_input(name, desc, index, stage, manifest)
@@ -319,6 +354,7 @@ def validate(manifest: Dict, index: Index) -> ValidationResult:
         res = schema.validate(stage)
         result.merge(res, path=path)
 
+        validate_stage_modules("Device", stage, path)
         validate_stage_modules("Input", stage, path)
 
     def validate_pipeline(pipeline, path):

osbuild/meta.py

@@ -281,6 +281,7 @@ class ModuleInfo:
     # Known modules and their corresponding directory name
     MODULES = {
         "Assembler": "assemblers",
+        "Device": "devices",
         "Input": "inputs",
         "Source": "sources",
         "Stage": "stages",
@@ -330,6 +331,13 @@ class ModuleInfo:
                 **opts,
             }
             schema["required"] = [type_id]
+        elif self.type in ("Device", ):
+            schema["additionalProperties"] = True
+            opts = self._load_opts(version, "1")
+            schema["properties"] = {
+                "type": {"enum": [self.name]},
+                "options": opts
+            }
         else:
             opts = self._load_opts(version, "1")
             schema.update(opts)

osbuild/pipeline.py

@@ -10,6 +10,7 @@ from . import buildroot
 from . import host
 from . import objectstore
 from . import remoteloop
+from .devices import Device
 from .inputs import Input
 from .sources import Source
 from .util import osrelease
@@ -43,6 +44,7 @@ class Stage:
         self.options = options
         self.checkpoint = False
         self.inputs = {}
+        self.devices = {}
 
     @property
     def name(self):
@@ -65,6 +67,11 @@ class Stage:
         self.inputs[name] = ip
         return ip
 
+    def add_device(self, name, info, options):
+        dev = Device(name, info, options)
+        self.devices[name] = dev
+        return dev
+
     def run(self, tree, runner, build_tree, store, monitor, libdir):
         with contextlib.ExitStack() as cm:
 
@@ -76,12 +83,17 @@ class Stage:
             inputs_mapped = "/run/osbuild/inputs"
             inputs = {}
 
+            devices_mapped = "/dev"
+            devices = {}
+
             args = {
                 "tree": "/run/osbuild/tree",
                 "options": self.options,
                 "paths": {
+                    "devices": devices_mapped,
                     "inputs": inputs_mapped
                 },
+                "devices": devices,
                 "inputs": inputs,
                 "meta": {
                     "id": self.id
@@ -103,6 +115,10 @@ class Stage:
                 data = ip.map(mgr, storeapi, inputs_tmpdir)
                 inputs[key] = data
 
+            for key, dev in self.devices.items():
+                reply = dev.open(mgr, build_root.dev, tree)
+                devices[key] = reply
+
             api = API(args, monitor)
             build_root.register_api(api)
 

schemas/osbuild2.json

@@ -16,6 +16,26 @@
 
   "definitions": {
 
+    "devices": {
+      "title": "Collection of devices for a stage",
+      "additionalProperties": {
+        "$ref": "#/definitions/device"
+      }
+    },
+
+    "device": {
+      "title": "Device for a stage",
+      "additionalProperties": false,
+      "required": ["type", "options"],
+      "properties": {
+        "type": { "type": "string" },
+        "options": {
+          "type": "object",
+          "additionalProperties": true
+        }
+      }
+    },
+
     "inputs": {
       "title": "Collection of inputs for a stage",
       "additionalProperties": false,
@@ -102,6 +122,7 @@
       "additionalProperties": false,
       "properties": {
         "type": { "type": "string" },
+        "devices": { "$ref": "#/definitions/devices" },
         "inputs": {"$ref": "#/definitions/inputs" },
         "options": {
           "type": "object",