From 9b09ed9eb49149e6a2d9b3451e2aee96b5175b4c Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 21 Nov 2023 12:51:02 +0100 Subject: [PATCH] osbuild: allow json data to come from a `{stage}-meta.json` file Instead of always parsing the python stage to load meta information allow the user of a new `{stage}-meta.json` file. This is a first step towards allowing modules to be written in a different language than python. It also has some practical advantages: - slightly faster as it avoids calling python to output the schemas - easier to write schemas as this can be done in a real json editor now - more extensible in a future where stages maybe binaries with shlib dependencies that are only satisfied in the buildroot but not on the host --- osbuild/meta.py | 41 ++++++++++++++-- stages/org.osbuild.noop | 18 +------ stages/org.osbuild.noop-meta.json | 23 +++++++++ test/mod/test_meta.py | 79 +++++++++++++++++++++++++++++++ 4 files changed, 139 insertions(+), 22 deletions(-) create mode 100644 stages/org.osbuild.noop-meta.json diff --git a/osbuild/meta.py b/osbuild/meta.py index 50c7c6cf..56e401c6 100644 --- a/osbuild/meta.py +++ b/osbuild/meta.py @@ -409,6 +409,42 @@ class ModuleInfo: @classmethod def load(cls, root, klass, name) -> Optional["ModuleInfo"]: + base = cls.MODULES.get(klass) + if not base: + raise ValueError(f"Unsupported type: {klass}") + path = os.path.join(root, base, name) + + try: + return cls._load_from_json(path, klass, name) + except FileNotFoundError: + # should we print a deprecation warning here? + pass + return cls._load_from_py(path, klass, name) + + @classmethod + def _load_from_json(cls, path, klass, name) -> Optional["ModuleInfo"]: + # ideas welcome for a better filename/suffix :) + meta_json_suffix = "-meta.json" + with open(path + meta_json_suffix, encoding="utf-8") as fp: + meta = json.load(fp) + + long_description = meta.get("description", "no description provided") + if isinstance(long_description, list): + long_description = "\n".join(long_description) + + info = { + "schema": { + "1": meta.get("schema", {}), + "2": meta.get("schema_2", {}), + }, + "desc": meta.get("summary", "no summary provided"), + "info": long_description, + "caps": meta.get("capabilities", set()), + } + return cls(klass, name, path, info) + + @classmethod + def _load_from_py(cls, path, klass, name) -> Optional["ModuleInfo"]: names = ["SCHEMA", "SCHEMA_2", "CAPABILITIES"] def filter_type(lst, target): @@ -417,11 +453,6 @@ class ModuleInfo: def targets(a): return [t.id for t in filter_type(a.targets, ast.Name)] - base = cls.MODULES.get(klass) - if not base: - raise ValueError(f"Unsupported type: {klass}") - - path = os.path.join(root, base, name) try: with open(path, encoding="utf8") as f: data = f.read() diff --git a/stages/org.osbuild.noop b/stages/org.osbuild.noop index c44e7606..460bea24 100755 --- a/stages/org.osbuild.noop +++ b/stages/org.osbuild.noop @@ -2,8 +2,7 @@ """ Do Nothing -No-op stage. Prints a JSON dump of the options passed into this stage and -leaves the tree untouched. Useful for testing, debugging, and wasting time. +See "org.osbuild.noop-meta.json" for more details. """ @@ -12,21 +11,6 @@ import sys import osbuild.api -SCHEMA_2 = """ -"options": { - "additionalProperties": true -}, -"devices": { - "additionalProperties": true -}, -"inputs": { - "additionalProperties": true -}, -"mounts": { - "additionalProperties": true -} -""" - def main(_tree, inputs, options): print("Not doing anything with these options:", json.dumps(options)) diff --git a/stages/org.osbuild.noop-meta.json b/stages/org.osbuild.noop-meta.json new file mode 100644 index 00000000..aa9ab33a --- /dev/null +++ b/stages/org.osbuild.noop-meta.json @@ -0,0 +1,23 @@ +{ + "summary": "Do Nothing", + "description": [ + "No-op stage.\n", + "Prints a JSON dump of the options passed into this stage and ", + "leaves the tree untouched. Useful for testing, debugging, and ", + "wasting time." + ], + "schema_2": { + "options": { + "additionalProperties": true + }, + "devices": { + "additionalProperties": true + }, + "inputs": { + "additionalProperties": true + }, + "mounts": { + "additionalProperties": true + } + } +} diff --git a/test/mod/test_meta.py b/test/mod/test_meta.py index b7ced843..73254af7 100644 --- a/test/mod/test_meta.py +++ b/test/mod/test_meta.py @@ -1,4 +1,6 @@ import os +import pathlib +import textwrap import pytest @@ -168,3 +170,80 @@ def test_schema(): assert not res res = schema.validate([1, 2, 3]) assert res + + +def make_fake_meta_json(tmp_path, name): + meta_json_path = pathlib.Path(f"{tmp_path}/stages/{name}-meta.json") + meta_json_path.parent.mkdir(exist_ok=True) + meta_json_path.write_text(""" + { + "summary": "some json summary", + "description": [ + "long text", + "with newlines" + ], + "capabilities": ["CAP_MAC_ADMIN", "CAP_BIG_MAC"], + "schema": { + "properties": { + "json_filename": { + "type": "string" + } + } + }, + "schema_2": { + "json_devices": { + "type": "object" + } + } + } + """.replace("\n", " "), encoding="utf-8") + return meta_json_path + + +def make_fake_py_module(tmp_path, name): + py_path = pathlib.Path(f"{tmp_path}/stages/{name}") + py_path.parent.mkdir(exist_ok=True) + fake_py = '"""some py summary\nlong description\nwith newline"""' + fake_py += textwrap.dedent(""" + SCHEMA = '"properties": {"py_filename":{"type": "string"}}' + SCHEMA_2 = '"py_devices": {"type":"object"}' + CAPABILITIES = ['CAP_MAC_ADMIN'] + """) + py_path.write_text(fake_py, encoding="utf-8") + + +def test_load_from_json(tmp_path): + make_fake_meta_json(tmp_path, "org.osbuild.noop") + modinfo = osbuild.meta.ModuleInfo.load(tmp_path, "Stage", "org.osbuild.noop") + assert modinfo.desc == "some json summary" + assert modinfo.info == "long text\nwith newlines" + assert modinfo.caps == ["CAP_MAC_ADMIN", "CAP_BIG_MAC"] + assert modinfo.opts == { + "1": {"properties": {"json_filename": {"type": "string"}}}, + "2": {"json_devices": {"type": "object"}}, + } + + +def test_load_from_py(tmp_path): + make_fake_py_module(tmp_path, "org.osbuild.noop") + modinfo = osbuild.meta.ModuleInfo.load(tmp_path, "Stage", "org.osbuild.noop") + assert modinfo.desc == "some py summary" + assert modinfo.info == "long description\nwith newline" + assert modinfo.caps == set(["CAP_MAC_ADMIN"]) + assert modinfo.opts == { + "1": {"properties": {"py_filename": {"type": "string"}}}, + "2": {"py_devices": {"type": "object"}}, + } + + +def test_load_from_json_prefered(tmp_path): + make_fake_meta_json(tmp_path, "org.osbuild.noop") + make_fake_py_module(tmp_path, "org.osbuild.noop") + modinfo = osbuild.meta.ModuleInfo.load(tmp_path, "Stage", "org.osbuild.noop") + assert modinfo.desc == "some json summary" + assert modinfo.info == "long text\nwith newlines" + assert modinfo.caps == ["CAP_MAC_ADMIN", "CAP_BIG_MAC"] + assert modinfo.opts == { + "1": {"properties": {"json_filename": {"type": "string"}}}, + "2": {"json_devices": {"type": "object"}}, + }