osbuild: introduce sources

Pipelines encode which source content they need in the form of
repository metadata checksums (or rpm checksums). In addition, they
encode where they fetch that source content from in the form of URLs.
This is overly specific and doesn't have to be in the pipeline's hash:
the checksum is enough to specify an image.

In practice, this precluded using alternative ways of getting at source
packages, such as local mirrors, which could speed up development.

Introduce a new osbuild API: sources. With it, a stage can query for a
way to fetch source content based on checksums.

The first such source is `org.osbuild.dnf`, which returns repository
configuration for a metadata checksum. Note that the dnf stage continues
to verify that the content it received matches the checksum it expects.

Sources are implemented as programs, living in a `sources` directory.
They are run on the host (i.e., uncontained) right now. Each source gets
passed options, which are taken from a new command line argument to
osbuild, and an array of checksums for which to return content.

This API is only available to stages right now.

stages/org.osbuild.dnf

@@ -1,4 +1,5 @@
 #!/usr/bin/python3
+
 import contextlib
 import hashlib
 import json
@@ -8,6 +9,8 @@ import subprocess
 import sys
 import tempfile
 
+import osbuild.sources
+
 STAGE_DESC = "Install packages using DNF"
 STAGE_INFO = """
 Depsolves, downloads, and installs packages (and dependencies) using DNF.
@@ -54,39 +57,46 @@ STAGE_OPTS = """
 "required": ["repos", "packages", "releasever", "basearch"],
 "properties": {
   "repos": {
-    "description": "Array of repo objects to set up",
+    "description": "Array of checksums to repository sources or repo objects",
     "type": "array",
     "minItems": 1,
     "items": {
-        "type": "object",
-        "properties": {
-            "metalink": {
-                "description": "metalink URL for this repo",
-                "type": "string"
-            },
-            "mirrorlist": {
-                "description": "mirrorlist URL for this repo",
-                "type": "string"
-            },
-            "baseurl": {
-                "description": "baseurl for this repo",
-                "type": "string"
-            },
-            "checksum": {
-                "description": "checksum for the expected repo metadata",
-                "type": "string",
-                "pattern": "sha256:[a-fA-F0-9]{32}"
-            },
-            "gpgkey": {
-                "description": "GPG public key contents (to check signatures)",
-                "type": "string"
-            }
+      "anyOf": [
+        {
+          "type": "object",
+          "properties": {
+              "metalink": {
+                  "description": "metalink URL for this repo",
+                  "type": "string"
+              },
+              "mirrorlist": {
+                  "description": "mirrorlist URL for this repo",
+                  "type": "string"
+              },
+              "baseurl": {
+                  "description": "baseurl for this repo",
+                  "type": "string"
+              },
+              "checksum": {
+                  "description": "checksum for the expected repo metadata",
+                  "type": "string",
+                  "pattern": "sha256:[a-fA-F0-9]{32}"
+              },
+              "gpgkey": {
+                  "description": "GPG public key contents (to check signatures)",
+                  "type": "string"
+              }
+          },
+          "anyOf": [
+              {"required": ["checksum", "metalink"]},
+              {"required": ["checksum", "mirrorlist"]},
+              {"required": ["checksum", "baseurl"]}
+          ]
         },
-        "anyOf": [
-            {"required": ["checksum", "metalink"]},
-            {"required": ["checksum", "mirrorlist"]},
-            {"required": ["checksum", "baseurl"]}
-        ]
+        {
+          "type": "string"
+        }
+      ]
     }
   },
   "packages": {
@@ -126,6 +136,21 @@ STAGE_OPTS = """
 }
 """
 
+def fetch_repos(repos):
+    """Use osbuild's source api to fill in details about those repos in @repos
+    that only have a "checksum" set. All others are passed through unchanged.
+    """
+    checksums = []
+    result = []
+    for repo in repos:
+        if isinstance(repo, str):
+            checksums.append(repo)
+        else:
+            result.append(repo)
+
+    return result + osbuild.sources.get("org.osbuild.dnf", checksums)
+
+
 def write_repofile(f, repoid, repo, keydir):
     f.write(f"[{repoid}]\n")
 
@@ -172,7 +197,7 @@ def dnf_cachedir(repoid, repo, releasever, basearch):
 
 
 def main(tree, options):
-    repos = options["repos"]
+    repos = fetch_repos(options["repos"])
     packages = options["packages"]
     releasever = options["releasever"]
     basearch = options["basearch"]
@@ -181,6 +206,8 @@ def main(tree, options):
     exclude_packages = options.get("exclude_packages", [])
     module_platform_id = options.get("module_platform_id", None)
 
+    print("Repositories: {}\n".format(json.dumps(repos, indent=2)))
+
     script = f"""
         set -e
         mkdir -p {tree}/dev {tree}/sys {tree}/proc
@@ -266,7 +293,6 @@ def main(tree, options):
 
     return 0
 
-
 if __name__ == '__main__':
     args = json.load(sys.stdin)
     r = main(args["tree"], args["options"])