particle-os/debian-forge
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

osbuild: introduce mount host service

Browse source 
Allows stages to access file systems provided by devices.
This makes mount handling transparent to the stages, i.e.
the individual stages do not need any code for different
file system types and the underlying devices.
This commit is contained in:
Christian Kellner 2021-06-07 18:50:10 +02:00 • committed by Tom Gundersen
parent 92f936e15c
commit 367a044453
5 changed files with 227 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

47
osbuild/formats/v2.py
View file

@ -12,6 +12,7 @@ from ..sources import Source
VERSION = "2"
# pylint: disable=too-many-statements
def describe(manifest: Manifest, *, with_id=False) -> Dict:
# Undo the build, runner pairing introduce by the loading
@ -72,6 +73,24 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
}
return desc
def describe_mount(mnt):
desc = {
"type": mnt.info.name,
"device": mnt.device.name,
"target": mnt.target
}
if mnt.options:
desc["options"] = mnt.options
return desc
def describe_mounts(mounts: Dict):
desc = {
name: describe_mount(mnt)
for name, mnt in mounts.items()
}
return desc
def describe_stage(s: Stage):
desc = {
"type": s.info.name
@ -87,6 +106,10 @@ def describe(manifest: Manifest, *, with_id=False) -> Dict:
if devs:
desc["devices"] = devs
mounts = describe_mounts(s.mounts)
if mounts:
desc["mounts"] = mounts
ips = describe_inputs(s.inputs)
if ips:
desc["inputs"] = ips
@ -187,6 +210,22 @@ def load_input(name: str, description: Dict, index: Index, stage: Stage, manifes
ip.add_reference(r, desc)
def load_mount(name: str, description: Dict, index: Index, stage: Stage):
mount_type = description["type"]
info = index.get_module_info("Mount", mount_type)
source = description["source"]
target = description["target"]
options = description.get("options", {})
device = stage.devices.get(source)
if not device:
raise ValueError(f"Unknown device '{source}' for mount '{name}'")
stage.add_mount(name, info, device, target, options)
def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Manifest):
stage_type = description["type"]
opts = description.get("options", {})
@ -202,6 +241,10 @@ def load_stage(description: Dict, index: Index, pipeline: Pipeline, manifest: Ma
for name, desc in ips.items():
load_input(name, desc, index, stage, manifest)
mounts = description.get("mounts", {})
for name, desc in mounts.items():
load_mount(name, desc, index, stage)
return stage
@ -354,8 +397,8 @@ def validate(manifest: Dict, index: Index) -> ValidationResult:
res = schema.validate(stage)
result.merge(res, path=path)
validate_stage_modules("Device", stage, path)
validate_stage_modules("Input", stage, path)
for mod in ("Device", "Input", "Mount"):
validate_stage_modules(mod, stage, path)
def validate_pipeline(pipeline, path):
stages = pipeline.get("stages", [])

3
osbuild/meta.py
View file

@ -283,6 +283,7 @@ class ModuleInfo:
"Assembler": "assemblers",
"Device": "devices",
"Input": "inputs",
"Mount": "mounts",
"Source": "sources",
"Stage": "stages",
}
@ -331,7 +332,7 @@ class ModuleInfo:
**opts,
}
schema["required"] = [type_id]
elif self.type in ("Device", ):
elif self.type in ("Device", "Mount"):
schema["additionalProperties"] = True
opts = self._load_opts(version, "1")
schema["properties"] = {

122
osbuild/mounts.py Normal file
View file

@ -0,0 +1,122 @@
"""
Mount Handling for pipeline stages
Allows stages to access file systems provided by devices.
This makes mount handling transparent to the stages, i.e.
the individual stages do not need any code for different
file system types and the underlying devices.
"""
import abc
import hashlib
import json
import os
import subprocess
from typing import Dict, Tuple
from osbuild import host
class Mount:
"""
A single mount with its corresponding options
"""
def __init__(self, name, info, device, target, options: Dict):
self.name = name
self.info = info
self.device = device
self.target = target
self.options = options
self.id = self.calc_id()
def calc_id(self):
m = hashlib.sha256()
m.update(json.dumps(self.info.name, sort_keys=True).encode())
m.update(json.dumps(self.device.id, sort_keys=True).encode())
m.update(json.dumps(self.target, sort_keys=True).encode())
m.update(json.dumps(self.options, sort_keys=True).encode())
return m.hexdigest()
def mount(self, mgr: host.ServiceManager, dev: str, root: str) -> Tuple[Dict]:
args = {
"source": dev,
"root": root,
"target": self.target,
"options": self.options,
}
client = mgr.start(f"mount/{self.name}", self.info.path)
path = client.call("mount", args)
if not path.startswith(root):
raise RuntimeError(f"returned path '{path}' has wrong prefix")
path = os.path.relpath(path, root)
return {"path": path}
class MountService(host.Service):
"""Mount host service"""
def __init__(self, args):
super().__init__(args)
self.mountpoint = None
self.check = False
@abc.abstractmethod
def translate_options(self, options: Dict):
return []
def mount(self, source: str, root: str, target: str, options: Dict):
mountpoint = os.path.join(root, target.lstrip("/"))
args = self.translate_options(options)
os.makedirs(mountpoint, exist_ok=True)
self.mountpoint = mountpoint
subprocess.run(
["mount"] +
args + [
"--source", source,
"--target", mountpoint
],
check=True)
self.check = True
return mountpoint
def umount(self):
if not self.mountpoint:
return
self.sync()
print("umounting")
# We ignore errors here on purpose
subprocess.run(["umount", self.mountpoint],
check=self.check)
self.mountpoint = None
def sync(self):
subprocess.run(["sync", "-f", self.mountpoint],
check=self.check)
def stop(self):
self.umount()
def dispatch(self, method: str, args, _fds):
if method == "mount":
r = self.mount(args["source"],
args["root"],
args["target"],
args["options"])
return r, None
raise host.ProtocolError("Unknown method")

35
osbuild/pipeline.py
View file

@ -12,6 +12,7 @@ from . import objectstore
from . import remoteloop
from .devices import Device
from .inputs import Input
from .mounts import Mount
from .sources import Source
from .util import osrelease
@ -45,6 +46,7 @@ class Stage:
self.checkpoint = False
self.inputs = {}
self.devices = {}
self.mounts = {}
@property
def name(self):
@ -72,10 +74,16 @@ class Stage:
self.devices[name] = dev
return dev
def add_mount(self, name, info, device, target, options):
mount = Mount(name, info, device, target, options)
self.mounts[name] = mount
return mount
def run(self, tree, runner, build_tree, store, monitor, libdir):
with contextlib.ExitStack() as cm:
build_root = buildroot.BuildRoot(build_tree, runner, libdir, store.tmp)
build_root = buildroot.BuildRoot(
build_tree, runner, libdir, store.tmp)
cm.enter_context(build_root)
inputs_tmpdir = store.tempdir(prefix="inputs-")
@ -86,15 +94,22 @@ class Stage:
devices_mapped = "/dev"
devices = {}
mounts_tmpdir = store.tempdir(prefix="mounts-")
mounts_tmpdir = cm.enter_context(mounts_tmpdir)
mounts_mapped = "/run/osbuild/mounts"
mounts = {}
args = {
"tree": "/run/osbuild/tree",
"options": self.options,
"paths": {
"devices": devices_mapped,
"inputs": inputs_mapped
"inputs": inputs_mapped,
"mounts": mounts_mapped,
},
"devices": devices,
"inputs": inputs,
"mounts": mounts,
"meta": {
"id": self.id
}
@ -105,6 +120,11 @@ class Stage:
f"{inputs_tmpdir}:{inputs_mapped}"
]
binds = [
os.fspath(tree) + ":/run/osbuild/tree",
f"{mounts_tmpdir}:{mounts_mapped}"
]
storeapi = objectstore.StoreServer(store)
cm.enter_context(storeapi)
@ -119,6 +139,12 @@ class Stage:
reply = dev.open(mgr, build_root.dev, tree)
devices[key] = reply
for key, mount in self.mounts.items():
relpath = devices[mount.device.name]["path"]
abspath = os.path.join(build_root.dev, relpath)
data = mount.mount(mgr, abspath, mounts_tmpdir)
mounts[key] = data
api = API(args, monitor)
build_root.register_api(api)
@ -127,7 +153,7 @@ class Stage:
r = build_root.run([f"/run/osbuild/bin/{self.name}"],
monitor,
binds=[os.fspath(tree) + ":/run/osbuild/tree"],
binds=binds,
readonly_binds=ro_binds)
return BuildResult(self, r.returncode, r.output, api.metadata, api.error)
@ -158,7 +184,8 @@ class Pipeline:
return self.stages[-1].id if self.stages else None
def add_stage(self, info, options, sources_options=None):
stage = Stage(info, sources_options, self.build, self.id, options or {})
stage = Stage(info, sources_options, self.build,
self.id, options or {})
self.stages.append(stage)
if self.assembler:
self.assembler.base = stage.id

27
schemas/osbuild2.json
View file

@ -61,6 +61,32 @@
}
},
"mounts": {
"title": "Collection of mount points for a stage",
"additionalProperties": {
"$ref": "#/definitions/mount"
}
},
"mount": {
"title": "Mount point for a stage",
"additionalProperties": false,
"required": ["type", "source", "target"],
"properties": {
"type": { "type": "string" },
"source": {
"type": "string"
},
"target": {
"type": "string"
},
"options": {
"type": "object",
"additionalProperties": true
}
}
},
"pipelines": {
"title": "Collection of pipelines to execute",
"description": "Array of pipelines to execute one after another",
@ -124,6 +150,7 @@
"type": { "type": "string" },
"devices": { "$ref": "#/definitions/devices" },
"inputs": {"$ref": "#/definitions/inputs" },
"mounts": {"$ref": "#/definitions/mounts" },
"options": {
"type": "object",
"additionalProperties": true