osbuild: introduce mount host service

Allows stages to access file systems provided by devices.
This makes mount handling transparent to the stages, i.e.
the individual stages do not need any code for different
file system types and the underlying devices.

osbuild/formats/v2.py

@@ -12,6 +12,7 @@ from ..sources import Source
 VERSION = "2"
 
 
+# pylint: disable=too-many-statements
 def describe(manifest: Manifest, *, with_id=False) -> Dict:
 
     # Undo the build, runner pairing introduce by the loading
@@ -72,6 +73,24 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
         }
         return desc
 
+    def describe_mount(mnt):
+        desc = {
+            "type": mnt.info.name,
+            "device": mnt.device.name,
+            "target": mnt.target
+        }
+
+        if mnt.options:
+            desc["options"] = mnt.options
+        return desc
+
+    def describe_mounts(mounts: Dict):
+        desc = {
+            name: describe_mount(mnt)
+            for name, mnt in mounts.items()
+        }
+        return desc
+
     def describe_stage(s: Stage):
         desc = {
             "type": s.info.name
@@ -87,6 +106,10 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
         if devs:
             desc["devices"] = devs
 
+        mounts = describe_mounts(s.mounts)
+        if mounts:
+            desc["mounts"] = mounts
+
         ips = describe_inputs(s.inputs)
         if ips:
             desc["inputs"] = ips
@@ -187,6 +210,22 @@ def load_input(name: str, description: Dict, index: Index, stage: Stage, manifes
         ip.add_reference(r, desc)
 
 
+def load_mount(name: str, description: Dict, index: Index, stage: Stage):
+    mount_type = description["type"]
+    info = index.get_module_info("Mount", mount_type)
+
+    source = description["source"]
+    target = description["target"]
+
+    options = description.get("options", {})
+
+    device = stage.devices.get(source)
+    if not device:
+        raise ValueError(f"Unknown device '{source}' for mount '{name}'")
+
+    stage.add_mount(name, info, device, target, options)
+
+
 def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Manifest):
     stage_type = description["type"]
     opts = description.get("options", {})
@@ -202,6 +241,10 @@ def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Ma
     for name, desc in ips.items():
         load_input(name, desc, index, stage, manifest)
 
+    mounts = description.get("mounts", {})
+    for name, desc in mounts.items():
+        load_mount(name, desc, index, stage)
+
     return stage
 
 
@@ -354,8 +397,8 @@ def validate(manifest: Dict, index: Index) -> ValidationResult:
         res = schema.validate(stage)
         result.merge(res, path=path)
 
-        validate_stage_modules("Device", stage, path)
+        for mod in ("Device", "Input", "Mount"):
-        validate_stage_modules("Input", stage, path)
+            validate_stage_modules(mod, stage, path)
 
     def validate_pipeline(pipeline, path):
         stages = pipeline.get("stages", [])

osbuild/meta.py

@@ -283,6 +283,7 @@ class ModuleInfo:
         "Assembler": "assemblers",
         "Device": "devices",
         "Input": "inputs",
+        "Mount": "mounts",
         "Source": "sources",
         "Stage": "stages",
     }
@@ -331,7 +332,7 @@ class ModuleInfo:
                 **opts,
             }
             schema["required"] = [type_id]
-        elif self.type in ("Device", ):
+        elif self.type in ("Device", "Mount"):
             schema["additionalProperties"] = True
             opts = self._load_opts(version, "1")
             schema["properties"] = {

osbuild/mounts.py

@@ -0,0 +1,122 @@
+"""
+Mount Handling for pipeline stages
+
+Allows stages to access file systems provided by devices.
+This makes mount handling transparent to the stages, i.e.
+the individual stages do not need any code for different
+file system types and the underlying devices.
+"""
+
+import abc
+import hashlib
+import json
+import os
+import subprocess
+
+from typing import Dict, Tuple
+
+from osbuild import host
+
+
+class Mount:
+    """
+    A single mount with its corresponding options
+    """
+
+    def __init__(self, name, info, device, target, options: Dict):
+        self.name = name
+        self.info = info
+        self.device = device
+        self.target = target
+        self.options = options
+        self.id = self.calc_id()
+
+    def calc_id(self):
+        m = hashlib.sha256()
+        m.update(json.dumps(self.info.name, sort_keys=True).encode())
+        m.update(json.dumps(self.device.id, sort_keys=True).encode())
+        m.update(json.dumps(self.target, sort_keys=True).encode())
+        m.update(json.dumps(self.options, sort_keys=True).encode())
+        return m.hexdigest()
+
+    def mount(self, mgr: host.ServiceManager, dev: str, root: str) -> Tuple[Dict]:
+
+        args = {
+            "source": dev,
+            "root": root,
+            "target": self.target,
+
+            "options": self.options,
+        }
+
+        client = mgr.start(f"mount/{self.name}", self.info.path)
+        path = client.call("mount", args)
+
+        if not path.startswith(root):
+            raise RuntimeError(f"returned path '{path}' has wrong prefix")
+
+        path = os.path.relpath(path, root)
+        return {"path": path}
+
+
+class MountService(host.Service):
+    """Mount host service"""
+
+    def __init__(self, args):
+        super().__init__(args)
+
+        self.mountpoint = None
+        self.check = False
+
+    @abc.abstractmethod
+    def translate_options(self, options: Dict):
+        return []
+
+    def mount(self, source: str, root: str, target: str, options: Dict):
+
+        mountpoint = os.path.join(root, target.lstrip("/"))
+        args = self.translate_options(options)
+
+        os.makedirs(mountpoint, exist_ok=True)
+        self.mountpoint = mountpoint
+
+        subprocess.run(
+            ["mount"] +
+            args + [
+                "--source", source,
+                "--target", mountpoint
+            ],
+            check=True)
+
+        self.check = True
+        return mountpoint
+
+    def umount(self):
+        if not self.mountpoint:
+            return
+
+        self.sync()
+
+        print("umounting")
+
+        # We ignore errors here on purpose
+        subprocess.run(["umount", self.mountpoint],
+                       check=self.check)
+        self.mountpoint = None
+
+    def sync(self):
+        subprocess.run(["sync", "-f", self.mountpoint],
+                       check=self.check)
+
+    def stop(self):
+        self.umount()
+
+    def dispatch(self, method: str, args, _fds):
+        if method == "mount":
+            r = self.mount(args["source"],
+                           args["root"],
+                           args["target"],
+                           args["options"])
+            return r, None
+
+        raise host.ProtocolError("Unknown method")

osbuild/pipeline.py

@@ -12,6 +12,7 @@ from . import objectstore
 from . import remoteloop
 from .devices import Device
 from .inputs import Input
+from .mounts import Mount
 from .sources import Source
 from .util import osrelease
 
@@ -45,6 +46,7 @@ class Stage:
         self.checkpoint = False
         self.inputs = {}
         self.devices = {}
+        self.mounts = {}
 
     @property
     def name(self):
@@ -72,10 +74,16 @@ class Stage:
         self.devices[name] = dev
         return dev
 
+    def add_mount(self, name, info, device, target, options):
+        mount = Mount(name, info, device, target, options)
+        self.mounts[name] = mount
+        return mount
+
     def run(self, tree, runner, build_tree, store, monitor, libdir):
         with contextlib.ExitStack() as cm:
 
-            build_root = buildroot.BuildRoot(build_tree, runner, libdir, store.tmp)
+            build_root = buildroot.BuildRoot(
+                build_tree, runner, libdir, store.tmp)
             cm.enter_context(build_root)
 
             inputs_tmpdir = store.tempdir(prefix="inputs-")
@@ -86,15 +94,22 @@ class Stage:
             devices_mapped = "/dev"
             devices = {}
 
+            mounts_tmpdir = store.tempdir(prefix="mounts-")
+            mounts_tmpdir = cm.enter_context(mounts_tmpdir)
+            mounts_mapped = "/run/osbuild/mounts"
+            mounts = {}
+
             args = {
                 "tree": "/run/osbuild/tree",
                 "options": self.options,
                 "paths": {
                     "devices": devices_mapped,
-                    "inputs": inputs_mapped
+                    "inputs": inputs_mapped,
+                    "mounts": mounts_mapped,
                 },
                 "devices": devices,
                 "inputs": inputs,
+                "mounts": mounts,
                 "meta": {
                     "id": self.id
                 }
@@ -105,6 +120,11 @@ class Stage:
                 f"{inputs_tmpdir}:{inputs_mapped}"
             ]
 
+            binds = [
+                os.fspath(tree) + ":/run/osbuild/tree",
+                f"{mounts_tmpdir}:{mounts_mapped}"
+            ]
+
             storeapi = objectstore.StoreServer(store)
             cm.enter_context(storeapi)
 
@@ -119,6 +139,12 @@ class Stage:
                 reply = dev.open(mgr, build_root.dev, tree)
                 devices[key] = reply
 
+            for key, mount in self.mounts.items():
+                relpath = devices[mount.device.name]["path"]
+                abspath = os.path.join(build_root.dev, relpath)
+                data = mount.mount(mgr, abspath, mounts_tmpdir)
+                mounts[key] = data
+
             api = API(args, monitor)
             build_root.register_api(api)
 
@@ -127,7 +153,7 @@ class Stage:
 
             r = build_root.run([f"/run/osbuild/bin/{self.name}"],
                                monitor,
-                               binds=[os.fspath(tree) + ":/run/osbuild/tree"],
+                               binds=binds,
                                readonly_binds=ro_binds)
 
         return BuildResult(self, r.returncode, r.output, api.metadata, api.error)
@@ -158,7 +184,8 @@ class Pipeline:
         return self.stages[-1].id if self.stages else None
 
     def add_stage(self, info, options, sources_options=None):
-        stage = Stage(info, sources_options, self.build, self.id, options or {})
+        stage = Stage(info, sources_options, self.build,
+                      self.id, options or {})
         self.stages.append(stage)
         if self.assembler:
             self.assembler.base = stage.id

schemas/osbuild2.json

@@ -61,6 +61,32 @@
       }
     },
 
+    "mounts": {
+      "title": "Collection of mount points for a stage",
+      "additionalProperties": {
+        "$ref": "#/definitions/mount"
+      }
+    },
+
+    "mount": {
+      "title": "Mount point for a stage",
+      "additionalProperties": false,
+      "required": ["type", "source", "target"],
+      "properties": {
+        "type": { "type": "string" },
+        "source": {
+          "type": "string"
+        },
+        "target": {
+          "type": "string"
+        },
+        "options": {
+          "type": "object",
+          "additionalProperties": true
+        }
+      }
+    },
+
     "pipelines": {
       "title": "Collection of pipelines to execute",
       "description": "Array of pipelines to execute one after another",
@@ -124,6 +150,7 @@
         "type": { "type": "string" },
         "devices": { "$ref": "#/definitions/devices" },
         "inputs": {"$ref": "#/definitions/inputs" },
+        "mounts": {"$ref": "#/definitions/mounts" },
         "options": {
           "type": "object",
           "additionalProperties": true