tools/osbuild-mpp: add ability to embed data

Add a new pre-processor directive to MPP that allows to define
`org.osbuild.inline` sources on the fly by embedding data or
files into the manifest.

tools/osbuild-mpp

@@ -188,12 +188,59 @@ Example:
 ...
 ```
 
+Embedding data and files so they can be used in inputs:
+
+This directive allows to generate `org.osbuild.inline` sources on the fly. They can
+be generated by reading a file (via the `path` parameter) or by directly providing
+the data (via the `text` parameter). The reference to the inline source will be
+added to the array of references of the corresponding input. Any JSON specified
+via the `options` parameter will be passed as value for the reference. Additionally,
+a dictionary called `embedded` will be created and within a mapping from the `id` to
+the checksum so that the source can be used in e.g. `mpp-format-string` directvies.
+
+Example:
+
+```
+...
+    stages": [
+        {
+          "type": "org.osbuild.copy",
+          "inputs": {
+            "inlinefile": {
+              "type": "org.osbuild.files",
+              "origin": "org.osbuild.source",
+              "mpp-embed": {
+                "id": "hw",
+                "text": "Hallo Welt\n"
+              },
+              "references": {
+                ...
+              }
+            }
+          },
+          "options": {
+            "paths": [
+              {
+                "from": {
+                    "mpp-format-string": "input://inlinefile/{embedded['hw']}"
+                },
+                "to": "tree:///testfile"
+              }
+            ]
+          }
+        }
+      ]
+...
+```
+
 """
 
 
 import argparse
+import base64
 import collections
 import contextlib
+import hashlib
 import json
 import os
 import pathlib
@@ -217,6 +264,11 @@ def element_enter(element, key, default):
     return element[key]
 
 
+class EmbeddedFile:
+    def __init__(self) -> None:
+        pass
+
+
 class PkgInfo:
     def __init__(self, checksum, name, evr, arch):
         self.checksum = checksum
@@ -887,6 +939,9 @@ class ManifestFileV1(ManifestFile):
             if "pipeline" in build:
                 self.process_depsolves(solver, build["pipeline"], depth+1)
 
+    def process_embed_files(self):
+        "Embedding files is not supported for v1 manifests"
+
 
 class ManifestFileV2(ManifestFile):
     def __init__(self, path, overrides, default_vars, data):
@@ -961,6 +1016,58 @@ class ManifestFileV2(ManifestFile):
             for stage in stages:
                 self._process_depsolve(solver, stage, name)
 
+    def process_embed_files(self):
+
+        class Embedded(collections.namedtuple("Embedded", ["id", "checksum"])):
+            def __str__(self):
+                return self.checksum
+
+        def embed_data(ip, mpp):
+            uid = mpp["id"]
+            path = mpp.get("path")
+            text = mpp.get("text")
+
+            if path and text:
+                raise ValueError(f"Cannot specify both 'path' and 'text' for '{uid}'")
+
+            if path:
+                with self.find_and_open_file(path, [], mode="rb") as f:
+                    data = f.read()
+            else:
+                data = bytes(text, "utf-8")
+
+            encoded = base64.b64encode(data).decode("utf-8")
+            checksum = hashlib.sha256(data).hexdigest()
+            digest = "sha256:" + checksum
+
+            source = element_enter(self.sources, "org.osbuild.inline", {})
+            items = element_enter(source, "items", {})
+            items[digest] = {
+                "encoding": "base64",
+                "data": encoded
+            }
+
+            refs = element_enter(ip, "references", {})
+            refs[digest] = mpp.get("options", {})
+            ef = element_enter(self.vars, "embedded", {})
+            ef[uid] = Embedded(uid, digest)
+
+        for pipeline in self.pipelines:
+            for stage in pipeline.get("stages", []):
+                for ip in stage.get("inputs", {}).values():
+
+                    if ip["type"] != "org.osbuild.files":
+                        continue
+
+                    if ip["origin"] != "org.osbuild.source":
+                        continue
+
+                    mpp = self.get_mpp_node(ip, "embed")
+                    if not mpp:
+                        continue
+
+                    embed_data(ip, mpp)
+
 
 def main():
     parser = argparse.ArgumentParser(description="Manifest pre processor")
@@ -1024,6 +1131,8 @@ def main():
     # First resolve all imports
     m.process_imports(args.searchdirs)
 
+    m.process_embed_files()
+
     m.process_partition()
 
     with tempfile.TemporaryDirectory() as persistdir: