From 08bc9ab7d8e4216c4518863accca282a9601000f Mon Sep 17 00:00:00 2001 From: Christian Kellner Date: Tue, 1 Jun 2021 17:03:38 +0200 Subject: [PATCH] inputs: pre-defined input paths Instead of bind-mounting each individual input into the container, create a temporary directory that is used by all inputs and bind- mount this to the well known location ("/run/osbuild/inputs"). The temporary directory is then passed to the input so that it can make the requested resources available relative to that directory. This is enforced by the common input handling code. Additionally, pass the well known input path via a new "paths" key to the arguments dictionary passed to the stage. --- inputs/org.osbuild.files | 8 ++++---- inputs/org.osbuild.noop | 10 ++++++---- inputs/org.osbuild.ostree | 6 +++--- inputs/org.osbuild.tree | 9 ++++----- osbuild/inputs.py | 18 +++++++++++++++--- osbuild/pipeline.py | 22 +++++++++++++--------- 6 files changed, 45 insertions(+), 28 deletions(-) diff --git a/inputs/org.osbuild.files b/inputs/org.osbuild.files index 5dc2d90f..c1f08a25 100755 --- a/inputs/org.osbuild.files +++ b/inputs/org.osbuild.files @@ -80,10 +80,10 @@ SCHEMA = r""" def main(): args = json.load(sys.stdin) refs = args["refs"] + target = args["target"] store = StoreClient(connect_to=args["api"]["store"]) source = store.source("org.osbuild.files") - output = store.mkdtemp(prefix="files-input-") for checksum in refs: try: @@ -91,7 +91,7 @@ def main(): [ "ln", f"{source}/{checksum}", - f"{output}/{checksum}", + f"{target}/{checksum}", ], check=True, ) @@ -100,9 +100,9 @@ def main(): return 1 reply = { - "path": output, + "path": target, "data": { - "refs": refs + "refs": refs } } diff --git a/inputs/org.osbuild.noop b/inputs/org.osbuild.noop index 84bb7f2b..87f6a358 100755 --- a/inputs/org.osbuild.noop +++ b/inputs/org.osbuild.noop @@ -8,9 +8,9 @@ it to the stage. import json +import os import sys - -from osbuild.objectstore import StoreClient +import uuid SCHEMA = """ @@ -21,10 +21,12 @@ SCHEMA = """ def main(): args = json.load(sys.stdin) refs = args["refs"] + target = args["target"] - store = StoreClient(connect_to=args["api"]["store"]) + uid = str(uuid.uuid4()) + path = os.path.join(target, uid) + os.makedirs(path) - path = store.mkdtemp(prefix="empty") data = {"path": path, "data": {"refs": refs}} json.dump(data, sys.stdout) return 0 diff --git a/inputs/org.osbuild.ostree b/inputs/org.osbuild.ostree index 34625d5a..6f4731d8 100755 --- a/inputs/org.osbuild.ostree +++ b/inputs/org.osbuild.ostree @@ -105,10 +105,10 @@ def export(checksums, cache, output): def main(): args = json.load(sys.stdin) refs = args["refs"] + target = args["target"] origin = args["origin"] store = StoreClient(connect_to=args["api"]["store"]) - output = store.mkdtemp(prefix="ostree-output") if origin == "org.osbuild.pipeline": for ref, options in refs.items(): @@ -116,10 +116,10 @@ def main(): with open(os.path.join(source, "compose.json"), "r") as f: compose = json.load(f) commit_id = compose["ostree-commit"] - export({commit_id: options}, source, output) + export({commit_id: options}, source, target) else: source = store.source("org.osbuild.ostree") - export(refs, source, output) + export(refs, source, target) return 0 diff --git a/inputs/org.osbuild.tree b/inputs/org.osbuild.tree index 62c3af5b..f98bae0d 100755 --- a/inputs/org.osbuild.tree +++ b/inputs/org.osbuild.tree @@ -60,6 +60,7 @@ def error(msg): def main(): args = json.load(sys.stdin) refs = args["refs"] + target = args["target"] # input verification *must* have been done via schema # verification. It is expected that origin is a pipeline @@ -68,15 +69,13 @@ def main(): store = StoreClient(connect_to=args["api"]["store"]) - if not pid: - path = store.mkdtemp(prefix="empty") - else: - path = store.read_tree(pid) + if pid: + path = store.read_tree_at(pid, target) if not path: error(f"Could not find pipeline with id '{pid}'") - json.dump({"path": path}, sys.stdout) + json.dump({"path": target}, sys.stdout) return 0 diff --git a/osbuild/inputs.py b/osbuild/inputs.py index f58d6290..c51c06aa 100644 --- a/osbuild/inputs.py +++ b/osbuild/inputs.py @@ -25,6 +25,7 @@ import subprocess from typing import Dict, Optional, Tuple +from osbuild.util.types import PathLike from .objectstore import StoreServer @@ -58,13 +59,19 @@ class Input: m.update(json.dumps(self.options, sort_keys=True).encode()) return m.hexdigest() - def run(self, storeapi: StoreServer) -> Tuple[str, Dict]: + def run(self, storeapi: StoreServer, root: PathLike) -> Tuple[str, Dict]: name = self.info.name + + target = os.path.join(root, self.name) + os.makedirs(target) + msg = { # mandatory bits "origin": self.origin, "refs": self.refs, + "target": target, + # global options "options": self.options, @@ -101,6 +108,11 @@ class Input: if r.returncode != 0: raise RuntimeError(f"{name}: error {r.returncode}") - path, data = reply["path"], reply.get("data", {}) + path = reply["path"] - return path, data + if not path.startswith(root): + raise RuntimeError(f"returned {path} has wrong prefix") + + reply["path"] = os.path.relpath(path, root) + + return reply diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py index 9174a574..1f6433eb 100644 --- a/osbuild/pipeline.py +++ b/osbuild/pipeline.py @@ -70,30 +70,34 @@ class Stage: build_root = buildroot.BuildRoot(build_tree, runner, libdir, store.tmp) cm.enter_context(build_root) + inputs_tmpdir = store.tempdir(prefix="inputs-") + inputs_tmpdir = cm.enter_context(inputs_tmpdir) + inputs_mapped = "/run/osbuild/inputs" + inputs = {} + args = { "tree": "/run/osbuild/tree", "options": self.options, - "inputs": {}, + "paths": { + "inputs": inputs_mapped + }, + "inputs": inputs, "meta": { "id": self.id } } ro_binds = [ - f"{self.info.path}:/run/osbuild/bin/{self.name}" + f"{self.info.path}:/run/osbuild/bin/{self.name}", + f"{inputs_tmpdir}:{inputs_mapped}" ] storeapi = objectstore.StoreServer(store) cm.enter_context(storeapi) for key, ip in self.inputs.items(): - path, data = ip.run(storeapi) - - # bind mount the returned path into the container - mapped = f"/run/osbuild/inputs/{key}" - ro_binds += [f"{path}:{mapped}"] - - args["inputs"][key] = {"path": mapped, "data": data} + data = ip.run(storeapi, inputs_tmpdir) + inputs[key] = data api = API(args, monitor) build_root.register_api(api)