osbuild: allow json data to come from a {stage}-meta.json file

Instead of always parsing the python stage to load meta information
allow the user of a new `{stage}-meta.json` file. This is a first
step towards allowing modules to be written in a different language
than python. It also has some practical advantages:
- slightly faster as it avoids calling python to output the schemas
- easier to write schemas as this can be done in a real json editor
  now
- more extensible in a future where stages maybe binaries with
  shlib dependencies that are only satisfied in the buildroot
  but not on the host

osbuild/meta.py

@@ -409,6 +409,42 @@ class ModuleInfo:
 
     @classmethod
     def load(cls, root, klass, name) -> Optional["ModuleInfo"]:
+        base = cls.MODULES.get(klass)
+        if not base:
+            raise ValueError(f"Unsupported type: {klass}")
+        path = os.path.join(root, base, name)
+
+        try:
+            return cls._load_from_json(path, klass, name)
+        except FileNotFoundError:
+            # should we print a deprecation warning here?
+            pass
+        return cls._load_from_py(path, klass, name)
+
+    @classmethod
+    def _load_from_json(cls, path, klass, name) -> Optional["ModuleInfo"]:
+        # ideas welcome for a better filename/suffix :)
+        meta_json_suffix = "-meta.json"
+        with open(path + meta_json_suffix, encoding="utf-8") as fp:
+            meta = json.load(fp)
+
+        long_description = meta.get("description", "no description provided")
+        if isinstance(long_description, list):
+            long_description = "\n".join(long_description)
+
+        info = {
+            "schema": {
+                "1": meta.get("schema", {}),
+                "2": meta.get("schema_2", {}),
+            },
+            "desc": meta.get("summary", "no summary provided"),
+            "info": long_description,
+            "caps": meta.get("capabilities", set()),
+        }
+        return cls(klass, name, path, info)
+
+    @classmethod
+    def _load_from_py(cls, path, klass, name) -> Optional["ModuleInfo"]:
         names = ["SCHEMA", "SCHEMA_2", "CAPABILITIES"]
 
         def filter_type(lst, target):
@@ -417,11 +453,6 @@ class ModuleInfo:
         def targets(a):
             return [t.id for t in filter_type(a.targets, ast.Name)]
 
-        base = cls.MODULES.get(klass)
-        if not base:
-            raise ValueError(f"Unsupported type: {klass}")
-
-        path = os.path.join(root, base, name)
         try:
             with open(path, encoding="utf8") as f:
                 data = f.read()

stages/org.osbuild.noop

@@ -2,8 +2,7 @@
 """
 Do Nothing
 
-No-op stage. Prints a JSON dump of the options passed into this stage and
+See "org.osbuild.noop-meta.json" for more details.
-leaves the tree untouched. Useful for testing, debugging, and wasting time.
 """
 
 
@@ -12,21 +11,6 @@ import sys
 
 import osbuild.api
 
-SCHEMA_2 = """
-"options": {
-  "additionalProperties": true
-},
-"devices": {
-  "additionalProperties": true
-},
-"inputs": {
-  "additionalProperties": true
-},
-"mounts": {
-  "additionalProperties": true
-}
-"""
-
 
 def main(_tree, inputs, options):
     print("Not doing anything with these options:", json.dumps(options))

stages/org.osbuild.noop-meta.json

@@ -0,0 +1,23 @@
+{
+    "summary": "Do Nothing",
+    "description": [
+        "No-op stage.\n",
+        "Prints a JSON dump of the options passed into this stage and ",
+        "leaves the tree untouched. Useful for testing, debugging, and ",
+        "wasting time."
+    ],
+    "schema_2": {
+        "options": {
+            "additionalProperties": true
+        },
+        "devices": {
+            "additionalProperties": true
+        },
+        "inputs": {
+            "additionalProperties": true
+        },
+        "mounts": {
+            "additionalProperties": true
+        }
+    }
+}

test/mod/test_meta.py

@@ -1,4 +1,6 @@
 import os
+import pathlib
+import textwrap
 
 import pytest
 
@@ -168,3 +170,80 @@ def test_schema():
     assert not res
     res = schema.validate([1, 2, 3])
     assert res
+
+
+def make_fake_meta_json(tmp_path, name):
+    meta_json_path = pathlib.Path(f"{tmp_path}/stages/{name}-meta.json")
+    meta_json_path.parent.mkdir(exist_ok=True)
+    meta_json_path.write_text("""
+    {
+      "summary": "some json summary",
+      "description": [
+        "long text",
+        "with newlines"
+      ],
+      "capabilities": ["CAP_MAC_ADMIN", "CAP_BIG_MAC"],
+      "schema": {
+        "properties": {
+          "json_filename": {
+            "type": "string"
+          }
+        }
+      },
+      "schema_2": {
+        "json_devices": {
+          "type": "object"
+        }
+      }
+    }
+    """.replace("\n", " "), encoding="utf-8")
+    return meta_json_path
+
+
+def make_fake_py_module(tmp_path, name):
+    py_path = pathlib.Path(f"{tmp_path}/stages/{name}")
+    py_path.parent.mkdir(exist_ok=True)
+    fake_py = '"""some py summary\nlong description\nwith newline"""'
+    fake_py += textwrap.dedent("""
+    SCHEMA = '"properties": {"py_filename":{"type": "string"}}'
+    SCHEMA_2 = '"py_devices": {"type":"object"}'
+    CAPABILITIES = ['CAP_MAC_ADMIN']
+    """)
+    py_path.write_text(fake_py, encoding="utf-8")
+
+
+def test_load_from_json(tmp_path):
+    make_fake_meta_json(tmp_path, "org.osbuild.noop")
+    modinfo = osbuild.meta.ModuleInfo.load(tmp_path, "Stage", "org.osbuild.noop")
+    assert modinfo.desc == "some json summary"
+    assert modinfo.info == "long text\nwith newlines"
+    assert modinfo.caps == ["CAP_MAC_ADMIN", "CAP_BIG_MAC"]
+    assert modinfo.opts == {
+        "1": {"properties": {"json_filename": {"type": "string"}}},
+        "2": {"json_devices": {"type": "object"}},
+    }
+
+
+def test_load_from_py(tmp_path):
+    make_fake_py_module(tmp_path, "org.osbuild.noop")
+    modinfo = osbuild.meta.ModuleInfo.load(tmp_path, "Stage", "org.osbuild.noop")
+    assert modinfo.desc == "some py summary"
+    assert modinfo.info == "long description\nwith newline"
+    assert modinfo.caps == set(["CAP_MAC_ADMIN"])
+    assert modinfo.opts == {
+        "1": {"properties": {"py_filename": {"type": "string"}}},
+        "2": {"py_devices": {"type": "object"}},
+    }
+
+
+def test_load_from_json_prefered(tmp_path):
+    make_fake_meta_json(tmp_path, "org.osbuild.noop")
+    make_fake_py_module(tmp_path, "org.osbuild.noop")
+    modinfo = osbuild.meta.ModuleInfo.load(tmp_path, "Stage", "org.osbuild.noop")
+    assert modinfo.desc == "some json summary"
+    assert modinfo.info == "long text\nwith newlines"
+    assert modinfo.caps == ["CAP_MAC_ADMIN", "CAP_BIG_MAC"]
+    assert modinfo.opts == {
+        "1": {"properties": {"json_filename": {"type": "string"}}},
+        "2": {"json_devices": {"type": "object"}},
+    }