From 931eac23c3036be18709cd5024b9fa9fc20479a9 Mon Sep 17 00:00:00 2001
From: Christian Kellner <christian@kellner.me>
Date: Tue, 9 Feb 2021 13:28:06 +0000
Subject: [PATCH] sources: introduce source items

All sources fetch various types of `items`, the specific nature
of which is dependent on the source type, but they are all
identifyable by a opaque identifier. In order for osbuild to
check that all the inputs that a stage needs are are indeed
contained in the manifest description, osbuild must learn what
ids are fetched by what source. This is done by standarzing
the common "items" part, i.e. the "id" -> "options for that id"
mapping that is common to all sources.
For the version 1 of the format, extract the files and ostree
the item information from the respective options.
Adapt the sources (files, ostree) so that they use the new items
information, but also fall back to the old style; the latter is
needed since the sources tests still uses the SourceServer.
---
 osbuild/formats/v1.py      | 15 ++++++++++++++-
 osbuild/pipeline.py        |  4 ++--
 osbuild/sources.py         |  5 ++++-
 sources/org.osbuild.files  | 13 +++++++++----
 sources/org.osbuild.ostree |  9 ++++++---
 5 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/osbuild/formats/v1.py b/osbuild/formats/v1.py
index e7e448d7..d769eda3 100644
--- a/osbuild/formats/v1.py
+++ b/osbuild/formats/v1.py
@@ -116,7 +116,20 @@ def load_stage(description: Dict, index: Index, pipeline: Pipeline):
 
 def load_source(name: str, description: Dict, index: Index, manifest: Manifest):
     info = index.get_module_info("Source", name)
-    manifest.add_source(info, description)
+
+    if name == "org.osbuild.files":
+        items = description["urls"]
+    elif name == "org.osbuild.ostree":
+        items = description["commits"]
+    else:
+        raise ValueError(f"Unknown source type: {name}")
+
+    # NB: the entries, i.e. `urls`, `commits` are left in the
+    # description dict, although the sources are not using
+    # it anymore. The reason is that it makes `describe` work
+    # without any special casing
+
+    manifest.add_source(info, items, description)
 
 
 def load_pipeline(description: Dict, index: Index, manifest: Manifest, n: int = 0) -> Pipeline:
diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py
index a8a7f7cb..422a0b40 100644
--- a/osbuild/pipeline.py
+++ b/osbuild/pipeline.py
@@ -270,8 +270,8 @@ class Manifest:
         self.pipelines[name] = pipeline
         return pipeline
 
-    def add_source(self, info, options: Dict) -> Source:
-        source = Source(info, options)
+    def add_source(self, info, items: List, options: Dict) -> Source:
+        source = Source(info, items, options)
         self.sources.append(source)
         return source
 
diff --git a/osbuild/sources.py b/osbuild/sources.py
index 235cc0db..90ca1de9 100644
--- a/osbuild/sources.py
+++ b/osbuild/sources.py
@@ -13,14 +13,16 @@ class Source:
     """
     A single source with is corresponding options.
     """
-    def __init__(self, info, options) -> None:
+    def __init__(self, info, items, options) -> None:
         self.info = info
+        self.items = items or {}
         self.options = options
 
     def download(self, store: ObjectStore, libdir: PathLike):
         source = self.info.name
         cache = os.path.join(store.store, "sources", source)
         msg = {
+            "items": self.items,
             "options": self.options,
             "cache": cache,
             "output": None,
@@ -69,6 +71,7 @@ class SourcesServer(api.BaseAPI):
 
     def _run_source(self, source, checksums):
         msg = {
+            "items": {},
             "options": self.options.get(source, {}),
             "cache": f"{self.cache}/{source}",
             "output": f"{self.output}/{source}",
diff --git a/sources/org.osbuild.files b/sources/org.osbuild.files
index 30ecd53a..ca54359a 100755
--- a/sources/org.osbuild.files
+++ b/sources/org.osbuild.files
@@ -218,11 +218,12 @@ def export(checksums, cache, output):
     return 0
 
 
-def main(options, checksums, cache, output):
-    urls = options.get("urls", {})
-
+def main(urls, options, checksums, cache, output):
     download_only = not output
 
+    if not urls:
+        urls = options.get("urls", [])
+
     if urls:
         if not checksums and download_only:
             checksums = [k for k, _ in urls.items()]
@@ -244,5 +245,9 @@ def main(options, checksums, cache, output):
 
 if __name__ == '__main__':
     args = json.load(sys.stdin)
-    r = main(args["options"], args["checksums"], args["cache"], args.get("output"))
+    r = main(args["items"],
+             args["options"],
+             args["checksums"],
+             args["cache"],
+             args.get("output"))
     sys.exit(r)
diff --git a/sources/org.osbuild.ostree b/sources/org.osbuild.ostree
index f4dfb4a6..0305d5e7 100755
--- a/sources/org.osbuild.ostree
+++ b/sources/org.osbuild.ostree
@@ -117,8 +117,10 @@ def export(checksums, cache, output):
     json.dump({}, sys.stdout)
 
 
-def main(options, checksums, cache, output):
-    commits = options["commits"]
+def main(commits, options, checksums, cache, output):
+
+    if not commits:
+        commits = options.get("commits", {})
 
     os.makedirs(cache, exist_ok=True)
     download(commits, checksums, cache)
@@ -135,7 +137,8 @@ def main(options, checksums, cache, output):
 
 if __name__ == '__main__':
     source_args = json.load(sys.stdin)
-    r = main(source_args["options"],
+    r = main(source_args["items"],
+             source_args["options"],
              source_args["checksums"],
              source_args["cache"],
              source_args["output"])