diff --git a/osbuild/formats/v2.py b/osbuild/formats/v2.py index ebcf154e..b179ac4f 100644 --- a/osbuild/formats/v2.py +++ b/osbuild/formats/v2.py @@ -12,6 +12,7 @@ from ..sources import Source VERSION = "2" +# pylint: disable=too-many-statements def describe(manifest: Manifest, *, with_id=False) -> Dict: # Undo the build, runner pairing introduce by the loading @@ -72,6 +73,24 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict: } return desc + def describe_mount(mnt): + desc = { + "type": mnt.info.name, + "device": mnt.device.name, + "target": mnt.target + } + + if mnt.options: + desc["options"] = mnt.options + return desc + + def describe_mounts(mounts: Dict): + desc = { + name: describe_mount(mnt) + for name, mnt in mounts.items() + } + return desc + def describe_stage(s: Stage): desc = { "type": s.info.name @@ -87,6 +106,10 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict: if devs: desc["devices"] = devs + mounts = describe_mounts(s.mounts) + if mounts: + desc["mounts"] = mounts + ips = describe_inputs(s.inputs) if ips: desc["inputs"] = ips @@ -187,6 +210,22 @@ def load_input(name: str, description: Dict, index: Index, stage: Stage, manifes ip.add_reference(r, desc) +def load_mount(name: str, description: Dict, index: Index, stage: Stage): + mount_type = description["type"] + info = index.get_module_info("Mount", mount_type) + + source = description["source"] + target = description["target"] + + options = description.get("options", {}) + + device = stage.devices.get(source) + if not device: + raise ValueError(f"Unknown device '{source}' for mount '{name}'") + + stage.add_mount(name, info, device, target, options) + + def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Manifest): stage_type = description["type"] opts = description.get("options", {}) @@ -202,6 +241,10 @@ def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Ma for name, desc in ips.items(): load_input(name, desc, index, stage, manifest) + mounts = description.get("mounts", {}) + for name, desc in mounts.items(): + load_mount(name, desc, index, stage) + return stage @@ -354,8 +397,8 @@ def validate(manifest: Dict, index: Index) -> ValidationResult: res = schema.validate(stage) result.merge(res, path=path) - validate_stage_modules("Device", stage, path) - validate_stage_modules("Input", stage, path) + for mod in ("Device", "Input", "Mount"): + validate_stage_modules(mod, stage, path) def validate_pipeline(pipeline, path): stages = pipeline.get("stages", []) diff --git a/osbuild/meta.py b/osbuild/meta.py index f0251a44..f761466a 100644 --- a/osbuild/meta.py +++ b/osbuild/meta.py @@ -283,6 +283,7 @@ class ModuleInfo: "Assembler": "assemblers", "Device": "devices", "Input": "inputs", + "Mount": "mounts", "Source": "sources", "Stage": "stages", } @@ -331,7 +332,7 @@ class ModuleInfo: **opts, } schema["required"] = [type_id] - elif self.type in ("Device", ): + elif self.type in ("Device", "Mount"): schema["additionalProperties"] = True opts = self._load_opts(version, "1") schema["properties"] = { diff --git a/osbuild/mounts.py b/osbuild/mounts.py new file mode 100644 index 00000000..55affc40 --- /dev/null +++ b/osbuild/mounts.py @@ -0,0 +1,122 @@ +""" +Mount Handling for pipeline stages + +Allows stages to access file systems provided by devices. +This makes mount handling transparent to the stages, i.e. +the individual stages do not need any code for different +file system types and the underlying devices. +""" + +import abc +import hashlib +import json +import os +import subprocess + +from typing import Dict, Tuple + +from osbuild import host + + +class Mount: + """ + A single mount with its corresponding options + """ + + def __init__(self, name, info, device, target, options: Dict): + self.name = name + self.info = info + self.device = device + self.target = target + self.options = options + self.id = self.calc_id() + + def calc_id(self): + m = hashlib.sha256() + m.update(json.dumps(self.info.name, sort_keys=True).encode()) + m.update(json.dumps(self.device.id, sort_keys=True).encode()) + m.update(json.dumps(self.target, sort_keys=True).encode()) + m.update(json.dumps(self.options, sort_keys=True).encode()) + return m.hexdigest() + + def mount(self, mgr: host.ServiceManager, dev: str, root: str) -> Tuple[Dict]: + + args = { + "source": dev, + "root": root, + "target": self.target, + + "options": self.options, + } + + client = mgr.start(f"mount/{self.name}", self.info.path) + path = client.call("mount", args) + + if not path.startswith(root): + raise RuntimeError(f"returned path '{path}' has wrong prefix") + + path = os.path.relpath(path, root) + return {"path": path} + + +class MountService(host.Service): + """Mount host service""" + + def __init__(self, args): + super().__init__(args) + + self.mountpoint = None + self.check = False + + @abc.abstractmethod + def translate_options(self, options: Dict): + return [] + + def mount(self, source: str, root: str, target: str, options: Dict): + + mountpoint = os.path.join(root, target.lstrip("/")) + args = self.translate_options(options) + + os.makedirs(mountpoint, exist_ok=True) + self.mountpoint = mountpoint + + subprocess.run( + ["mount"] + + args + [ + "--source", source, + "--target", mountpoint + ], + check=True) + + self.check = True + return mountpoint + + def umount(self): + if not self.mountpoint: + return + + self.sync() + + print("umounting") + + # We ignore errors here on purpose + subprocess.run(["umount", self.mountpoint], + check=self.check) + self.mountpoint = None + + def sync(self): + subprocess.run(["sync", "-f", self.mountpoint], + check=self.check) + + def stop(self): + self.umount() + + def dispatch(self, method: str, args, _fds): + if method == "mount": + r = self.mount(args["source"], + args["root"], + args["target"], + args["options"]) + return r, None + + raise host.ProtocolError("Unknown method") diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index d87004e7..44ec7195 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -12,6 +12,7 @@ from . import objectstore from . import remoteloop from .devices import Device from .inputs import Input +from .mounts import Mount from .sources import Source from .util import osrelease @@ -45,6 +46,7 @@ class Stage: self.checkpoint = False self.inputs = {} self.devices = {} + self.mounts = {} @property def name(self): @@ -72,10 +74,16 @@ class Stage: self.devices[name] = dev return dev + def add_mount(self, name, info, device, target, options): + mount = Mount(name, info, device, target, options) + self.mounts[name] = mount + return mount + def run(self, tree, runner, build_tree, store, monitor, libdir): with contextlib.ExitStack() as cm: - build_root = buildroot.BuildRoot(build_tree, runner, libdir, store.tmp) + build_root = buildroot.BuildRoot( + build_tree, runner, libdir, store.tmp) cm.enter_context(build_root) inputs_tmpdir = store.tempdir(prefix="inputs-") @@ -86,15 +94,22 @@ class Stage: devices_mapped = "/dev" devices = {} + mounts_tmpdir = store.tempdir(prefix="mounts-") + mounts_tmpdir = cm.enter_context(mounts_tmpdir) + mounts_mapped = "/run/osbuild/mounts" + mounts = {} + args = { "tree": "/run/osbuild/tree", "options": self.options, "paths": { "devices": devices_mapped, - "inputs": inputs_mapped + "inputs": inputs_mapped, + "mounts": mounts_mapped, }, "devices": devices, "inputs": inputs, + "mounts": mounts, "meta": { "id": self.id } @@ -105,6 +120,11 @@ class Stage: f"{inputs_tmpdir}:{inputs_mapped}" ] + binds = [ + os.fspath(tree) + ":/run/osbuild/tree", + f"{mounts_tmpdir}:{mounts_mapped}" + ] + storeapi = objectstore.StoreServer(store) cm.enter_context(storeapi) @@ -119,6 +139,12 @@ class Stage: reply = dev.open(mgr, build_root.dev, tree) devices[key] = reply + for key, mount in self.mounts.items(): + relpath = devices[mount.device.name]["path"] + abspath = os.path.join(build_root.dev, relpath) + data = mount.mount(mgr, abspath, mounts_tmpdir) + mounts[key] = data + api = API(args, monitor) build_root.register_api(api) @@ -127,7 +153,7 @@ class Stage: r = build_root.run([f"/run/osbuild/bin/{self.name}"], monitor, - binds=[os.fspath(tree) + ":/run/osbuild/tree"], + binds=binds, readonly_binds=ro_binds) return BuildResult(self, r.returncode, r.output, api.metadata, api.error) @@ -158,7 +184,8 @@ class Pipeline: return self.stages[-1].id if self.stages else None def add_stage(self, info, options, sources_options=None): - stage = Stage(info, sources_options, self.build, self.id, options or {}) + stage = Stage(info, sources_options, self.build, + self.id, options or {}) self.stages.append(stage) if self.assembler: self.assembler.base = stage.id diff --git a/schemas/osbuild2.json b/schemas/osbuild2.json index a1d8742b..d7e7beb0 100644 --- a/schemas/osbuild2.json +++ b/schemas/osbuild2.json @@ -61,6 +61,32 @@ } }, + "mounts": { + "title": "Collection of mount points for a stage", + "additionalProperties": { + "$ref": "#/definitions/mount" + } + }, + + "mount": { + "title": "Mount point for a stage", + "additionalProperties": false, + "required": ["type", "source", "target"], + "properties": { + "type": { "type": "string" }, + "source": { + "type": "string" + }, + "target": { + "type": "string" + }, + "options": { + "type": "object", + "additionalProperties": true + } + } + }, + "pipelines": { "title": "Collection of pipelines to execute", "description": "Array of pipelines to execute one after another", @@ -124,6 +150,7 @@ "type": { "type": "string" }, "devices": { "$ref": "#/definitions/devices" }, "inputs": {"$ref": "#/definitions/inputs" }, + "mounts": {"$ref": "#/definitions/mounts" }, "options": { "type": "object", "additionalProperties": true