sources: introduce source items

All sources fetch various types of `items`, the specific nature
of which is dependent on the source type, but they are all
identifyable by a opaque identifier. In order for osbuild to
check that all the inputs that a stage needs are are indeed
contained in the manifest description, osbuild must learn what
ids are fetched by what source. This is done by standarzing
the common "items" part, i.e. the "id" -> "options for that id"
mapping that is common to all sources.
For the version 1 of the format, extract the files and ostree
the item information from the respective options.
Adapt the sources (files, ostree) so that they use the new items
information, but also fall back to the old style; the latter is
needed since the sources tests still uses the SourceServer.

osbuild/formats/v1.py

@@ -116,7 +116,20 @@ def load_stage(description: Dict, index: Index, pipeline: Pipeline):
 
 def load_source(name: str, description: Dict, index: Index, manifest: Manifest):
     info = index.get_module_info("Source", name)
-    manifest.add_source(info, description)
+
+    if name == "org.osbuild.files":
+        items = description["urls"]
+    elif name == "org.osbuild.ostree":
+        items = description["commits"]
+    else:
+        raise ValueError(f"Unknown source type: {name}")
+
+    # NB: the entries, i.e. `urls`, `commits` are left in the
+    # description dict, although the sources are not using
+    # it anymore. The reason is that it makes `describe` work
+    # without any special casing
+
+    manifest.add_source(info, items, description)
 
 
 def load_pipeline(description: Dict, index: Index, manifest: Manifest, n: int = 0) -> Pipeline:

osbuild/pipeline.py

@@ -270,8 +270,8 @@ class Manifest:
         self.pipelines[name] = pipeline
         return pipeline
 
-    def add_source(self, info, options: Dict) -> Source:
-        source = Source(info, options)
+    def add_source(self, info, items: List, options: Dict) -> Source:
+        source = Source(info, items, options)
         self.sources.append(source)
         return source
 

osbuild/sources.py

@@ -13,14 +13,16 @@ class Source:
     """
     A single source with is corresponding options.
     """
-    def __init__(self, info, options) -> None:
+    def __init__(self, info, items, options) -> None:
         self.info = info
+        self.items = items or {}
         self.options = options
 
     def download(self, store: ObjectStore, libdir: PathLike):
         source = self.info.name
         cache = os.path.join(store.store, "sources", source)
         msg = {
+            "items": self.items,
             "options": self.options,
             "cache": cache,
             "output": None,
@@ -69,6 +71,7 @@ class SourcesServer(api.BaseAPI):
 
     def _run_source(self, source, checksums):
         msg = {
+            "items": {},
             "options": self.options.get(source, {}),
             "cache": f"{self.cache}/{source}",
             "output": f"{self.output}/{source}",

sources/org.osbuild.files

@@ -218,11 +218,12 @@ def export(checksums, cache, output):
     return 0
 
 
-def main(options, checksums, cache, output):
-    urls = options.get("urls", {})
-
+def main(urls, options, checksums, cache, output):
     download_only = not output
 
+    if not urls:
+        urls = options.get("urls", [])
+
     if urls:
         if not checksums and download_only:
             checksums = [k for k, _ in urls.items()]
@@ -244,5 +245,9 @@ def main(options, checksums, cache, output):
 
 if __name__ == '__main__':
     args = json.load(sys.stdin)
-    r = main(args["options"], args["checksums"], args["cache"], args.get("output"))
+    r = main(args["items"],
+             args["options"],
+             args["checksums"],
+             args["cache"],
+             args.get("output"))
     sys.exit(r)

sources/org.osbuild.ostree

@@ -117,8 +117,10 @@ def export(checksums, cache, output):
     json.dump({}, sys.stdout)
 
 
-def main(options, checksums, cache, output):
-    commits = options["commits"]
+def main(commits, options, checksums, cache, output):
+
+    if not commits:
+        commits = options.get("commits", {})
 
     os.makedirs(cache, exist_ok=True)
     download(commits, checksums, cache)
@@ -135,7 +137,8 @@ def main(options, checksums, cache, output):
 
 if __name__ == '__main__':
     source_args = json.load(sys.stdin)
-    r = main(source_args["options"],
+    r = main(source_args["items"],
+             source_args["options"],
              source_args["checksums"],
              source_args["cache"],
              source_args["output"])