From 367a044453da7738fe73f44ebc4319535143a68e Mon Sep 17 00:00:00 2001
From: Christian Kellner <christian@kellner.me>
Date: Mon, 7 Jun 2021 18:50:10 +0200
Subject: [PATCH] osbuild: introduce mount host service

Allows stages to access file systems provided by devices.
This makes mount handling transparent to the stages, i.e.
the individual stages do not need any code for different
file system types and the underlying devices.
---
 osbuild/formats/v2.py |  47 +++++++++++++++-
 osbuild/meta.py       |   3 +-
 osbuild/mounts.py     | 122 ++++++++++++++++++++++++++++++++++++++++++
 osbuild/pipeline.py   |  35 ++++++++++--
 schemas/osbuild2.json |  27 ++++++++++
 5 files changed, 227 insertions(+), 7 deletions(-)
 create mode 100644 osbuild/mounts.py

diff --git a/osbuild/formats/v2.py b/osbuild/formats/v2.py
index ebcf154e..b179ac4f 100644
--- a/osbuild/formats/v2.py
+++ b/osbuild/formats/v2.py
@@ -12,6 +12,7 @@ from ..sources import Source
 VERSION = "2"
 
 
+# pylint: disable=too-many-statements
 def describe(manifest: Manifest, *, with_id=False) -> Dict:
 
     # Undo the build, runner pairing introduce by the loading
@@ -72,6 +73,24 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
         }
         return desc
 
+    def describe_mount(mnt):
+        desc = {
+            "type": mnt.info.name,
+            "device": mnt.device.name,
+            "target": mnt.target
+        }
+
+        if mnt.options:
+            desc["options"] = mnt.options
+        return desc
+
+    def describe_mounts(mounts: Dict):
+        desc = {
+            name: describe_mount(mnt)
+            for name, mnt in mounts.items()
+        }
+        return desc
+
     def describe_stage(s: Stage):
         desc = {
             "type": s.info.name
@@ -87,6 +106,10 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
         if devs:
             desc["devices"] = devs
 
+        mounts = describe_mounts(s.mounts)
+        if mounts:
+            desc["mounts"] = mounts
+
         ips = describe_inputs(s.inputs)
         if ips:
             desc["inputs"] = ips
@@ -187,6 +210,22 @@ def load_input(name: str, description: Dict, index: Index, stage: Stage, manifes
         ip.add_reference(r, desc)
 
 
+def load_mount(name: str, description: Dict, index: Index, stage: Stage):
+    mount_type = description["type"]
+    info = index.get_module_info("Mount", mount_type)
+
+    source = description["source"]
+    target = description["target"]
+
+    options = description.get("options", {})
+
+    device = stage.devices.get(source)
+    if not device:
+        raise ValueError(f"Unknown device '{source}' for mount '{name}'")
+
+    stage.add_mount(name, info, device, target, options)
+
+
 def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Manifest):
     stage_type = description["type"]
     opts = description.get("options", {})
@@ -202,6 +241,10 @@ def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Ma
     for name, desc in ips.items():
         load_input(name, desc, index, stage, manifest)
 
+    mounts = description.get("mounts", {})
+    for name, desc in mounts.items():
+        load_mount(name, desc, index, stage)
+
     return stage
 
 
@@ -354,8 +397,8 @@ def validate(manifest: Dict, index: Index) -> ValidationResult:
         res = schema.validate(stage)
         result.merge(res, path=path)
 
-        validate_stage_modules("Device", stage, path)
-        validate_stage_modules("Input", stage, path)
+        for mod in ("Device", "Input", "Mount"):
+            validate_stage_modules(mod, stage, path)
 
     def validate_pipeline(pipeline, path):
         stages = pipeline.get("stages", [])
diff --git a/osbuild/meta.py b/osbuild/meta.py
index f0251a44..f761466a 100644
--- a/osbuild/meta.py
+++ b/osbuild/meta.py
@@ -283,6 +283,7 @@ class ModuleInfo:
         "Assembler": "assemblers",
         "Device": "devices",
         "Input": "inputs",
+        "Mount": "mounts",
         "Source": "sources",
         "Stage": "stages",
     }
@@ -331,7 +332,7 @@ class ModuleInfo:
                 **opts,
             }
             schema["required"] = [type_id]
-        elif self.type in ("Device", ):
+        elif self.type in ("Device", "Mount"):
             schema["additionalProperties"] = True
             opts = self._load_opts(version, "1")
             schema["properties"] = {
diff --git a/osbuild/mounts.py b/osbuild/mounts.py
new file mode 100644
index 00000000..55affc40
--- /dev/null
+++ b/osbuild/mounts.py
@@ -0,0 +1,122 @@
+"""
+Mount Handling for pipeline stages
+
+Allows stages to access file systems provided by devices.
+This makes mount handling transparent to the stages, i.e.
+the individual stages do not need any code for different
+file system types and the underlying devices.
+"""
+
+import abc
+import hashlib
+import json
+import os
+import subprocess
+
+from typing import Dict, Tuple
+
+from osbuild import host
+
+
+class Mount:
+    """
+    A single mount with its corresponding options
+    """
+
+    def __init__(self, name, info, device, target, options: Dict):
+        self.name = name
+        self.info = info
+        self.device = device
+        self.target = target
+        self.options = options
+        self.id = self.calc_id()
+
+    def calc_id(self):
+        m = hashlib.sha256()
+        m.update(json.dumps(self.info.name, sort_keys=True).encode())
+        m.update(json.dumps(self.device.id, sort_keys=True).encode())
+        m.update(json.dumps(self.target, sort_keys=True).encode())
+        m.update(json.dumps(self.options, sort_keys=True).encode())
+        return m.hexdigest()
+
+    def mount(self, mgr: host.ServiceManager, dev: str, root: str) -> Tuple[Dict]:
+
+        args = {
+            "source": dev,
+            "root": root,
+            "target": self.target,
+
+            "options": self.options,
+        }
+
+        client = mgr.start(f"mount/{self.name}", self.info.path)
+        path = client.call("mount", args)
+
+        if not path.startswith(root):
+            raise RuntimeError(f"returned path '{path}' has wrong prefix")
+
+        path = os.path.relpath(path, root)
+        return {"path": path}
+
+
+class MountService(host.Service):
+    """Mount host service"""
+
+    def __init__(self, args):
+        super().__init__(args)
+
+        self.mountpoint = None
+        self.check = False
+
+    @abc.abstractmethod
+    def translate_options(self, options: Dict):
+        return []
+
+    def mount(self, source: str, root: str, target: str, options: Dict):
+
+        mountpoint = os.path.join(root, target.lstrip("/"))
+        args = self.translate_options(options)
+
+        os.makedirs(mountpoint, exist_ok=True)
+        self.mountpoint = mountpoint
+
+        subprocess.run(
+            ["mount"] +
+            args + [
+                "--source", source,
+                "--target", mountpoint
+            ],
+            check=True)
+
+        self.check = True
+        return mountpoint
+
+    def umount(self):
+        if not self.mountpoint:
+            return
+
+        self.sync()
+
+        print("umounting")
+
+        # We ignore errors here on purpose
+        subprocess.run(["umount", self.mountpoint],
+                       check=self.check)
+        self.mountpoint = None
+
+    def sync(self):
+        subprocess.run(["sync", "-f", self.mountpoint],
+                       check=self.check)
+
+    def stop(self):
+        self.umount()
+
+    def dispatch(self, method: str, args, _fds):
+        if method == "mount":
+            r = self.mount(args["source"],
+                           args["root"],
+                           args["target"],
+                           args["options"])
+            return r, None
+
+        raise host.ProtocolError("Unknown method")
diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py
index d87004e7..44ec7195 100644
--- a/osbuild/pipeline.py
+++ b/osbuild/pipeline.py
@@ -12,6 +12,7 @@ from . import objectstore
 from . import remoteloop
 from .devices import Device
 from .inputs import Input
+from .mounts import Mount
 from .sources import Source
 from .util import osrelease
 
@@ -45,6 +46,7 @@ class Stage:
         self.checkpoint = False
         self.inputs = {}
         self.devices = {}
+        self.mounts = {}
 
     @property
     def name(self):
@@ -72,10 +74,16 @@ class Stage:
         self.devices[name] = dev
         return dev
 
+    def add_mount(self, name, info, device, target, options):
+        mount = Mount(name, info, device, target, options)
+        self.mounts[name] = mount
+        return mount
+
     def run(self, tree, runner, build_tree, store, monitor, libdir):
         with contextlib.ExitStack() as cm:
 
-            build_root = buildroot.BuildRoot(build_tree, runner, libdir, store.tmp)
+            build_root = buildroot.BuildRoot(
+                build_tree, runner, libdir, store.tmp)
             cm.enter_context(build_root)
 
             inputs_tmpdir = store.tempdir(prefix="inputs-")
@@ -86,15 +94,22 @@ class Stage:
             devices_mapped = "/dev"
             devices = {}
 
+            mounts_tmpdir = store.tempdir(prefix="mounts-")
+            mounts_tmpdir = cm.enter_context(mounts_tmpdir)
+            mounts_mapped = "/run/osbuild/mounts"
+            mounts = {}
+
             args = {
                 "tree": "/run/osbuild/tree",
                 "options": self.options,
                 "paths": {
                     "devices": devices_mapped,
-                    "inputs": inputs_mapped
+                    "inputs": inputs_mapped,
+                    "mounts": mounts_mapped,
                 },
                 "devices": devices,
                 "inputs": inputs,
+                "mounts": mounts,
                 "meta": {
                     "id": self.id
                 }
@@ -105,6 +120,11 @@ class Stage:
                 f"{inputs_tmpdir}:{inputs_mapped}"
             ]
 
+            binds = [
+                os.fspath(tree) + ":/run/osbuild/tree",
+                f"{mounts_tmpdir}:{mounts_mapped}"
+            ]
+
             storeapi = objectstore.StoreServer(store)
             cm.enter_context(storeapi)
 
@@ -119,6 +139,12 @@ class Stage:
                 reply = dev.open(mgr, build_root.dev, tree)
                 devices[key] = reply
 
+            for key, mount in self.mounts.items():
+                relpath = devices[mount.device.name]["path"]
+                abspath = os.path.join(build_root.dev, relpath)
+                data = mount.mount(mgr, abspath, mounts_tmpdir)
+                mounts[key] = data
+
             api = API(args, monitor)
             build_root.register_api(api)
 
@@ -127,7 +153,7 @@ class Stage:
 
             r = build_root.run([f"/run/osbuild/bin/{self.name}"],
                                monitor,
-                               binds=[os.fspath(tree) + ":/run/osbuild/tree"],
+                               binds=binds,
                                readonly_binds=ro_binds)
 
         return BuildResult(self, r.returncode, r.output, api.metadata, api.error)
@@ -158,7 +184,8 @@ class Pipeline:
         return self.stages[-1].id if self.stages else None
 
     def add_stage(self, info, options, sources_options=None):
-        stage = Stage(info, sources_options, self.build, self.id, options or {})
+        stage = Stage(info, sources_options, self.build,
+                      self.id, options or {})
         self.stages.append(stage)
         if self.assembler:
             self.assembler.base = stage.id
diff --git a/schemas/osbuild2.json b/schemas/osbuild2.json
index a1d8742b..d7e7beb0 100644
--- a/schemas/osbuild2.json
+++ b/schemas/osbuild2.json
@@ -61,6 +61,32 @@
       }
     },
 
+    "mounts": {
+      "title": "Collection of mount points for a stage",
+      "additionalProperties": {
+        "$ref": "#/definitions/mount"
+      }
+    },
+
+    "mount": {
+      "title": "Mount point for a stage",
+      "additionalProperties": false,
+      "required": ["type", "source", "target"],
+      "properties": {
+        "type": { "type": "string" },
+        "source": {
+          "type": "string"
+        },
+        "target": {
+          "type": "string"
+        },
+        "options": {
+          "type": "object",
+          "additionalProperties": true
+        }
+      }
+    },
+
     "pipelines": {
       "title": "Collection of pipelines to execute",
       "description": "Array of pipelines to execute one after another",
@@ -124,6 +150,7 @@
         "type": { "type": "string" },
         "devices": { "$ref": "#/definitions/devices" },
         "inputs": {"$ref": "#/definitions/inputs" },
+        "mounts": {"$ref": "#/definitions/mounts" },
         "options": {
           "type": "object",
           "additionalProperties": true