osbuild: introduce secrets

Add a new command line option `--secrets`, which accepts a JSON file
that is structured similarly to a source file. It is should contain data
that is necessary to fetch content, but shouldn't appear in any logs.

osbuild/__main__.py

@@ -21,6 +21,8 @@ def main():
                         help="the directory where intermediary os trees are stored")
     parser.add_argument("--sources", metavar="SOURCES", type=os.path.abspath,
                         help="json file containing a dictionary of source configuration")
+    parser.add_argument("--secrets", metavar="SECRETS", type=os.path.abspath,
+                        help="json file containing a dictionary of secrets that are passed to sources")
     parser.add_argument("-l", "--libdir", metavar="DIRECTORY", type=os.path.abspath,
                         help="the directory containing stages, assemblers, and the osbuild library")
     parser.add_argument("--json", action="store_true",
@@ -44,8 +46,19 @@ def main():
         with open(args.sources) as f:
             source_options = json.load(f)
 
+    secrets = {}
+    if args.secrets:
+        with open(args.secrets) as f:
+            secrets = json.load(f)
+
     try:
-        r = pipeline.run(args.store, interactive=not args.json, libdir=args.libdir, source_options=source_options)
+        r = pipeline.run(
+            args.store,
+            interactive=not args.json,
+            libdir=args.libdir,
+            source_options=source_options,
+            secrets=secrets
+        )
     except KeyboardInterrupt:
         print()
         print(f"{RESET}{BOLD}{RED}Aborted{RESET}")

osbuild/pipeline.py

@@ -68,7 +68,7 @@ class Stage:
             description["options"] = self.options
         return description
 
-    def run(self, tree, runner, build_tree, interactive=False, libdir=None, source_options=None):
+    def run(self, tree, runner, build_tree, interactive=False, libdir=None, source_options=None, secrets=None):
         with buildroot.BuildRoot(build_tree, runner, libdir=libdir) as build_root:
             if interactive:
                 print_header(f"{self.name}: {self.id}", self.options)
@@ -81,7 +81,7 @@ class Stage:
             sources_dir = f"{libdir}/sources" if libdir else "/usr/lib/osbuild/sources"
 
             with API(f"{build_root.api}/osbuild", args, interactive) as api, \
-                sources.SourcesServer(f"{build_root.api}/sources", sources_dir, source_options or {}):
+                sources.SourcesServer(f"{build_root.api}/sources", sources_dir, source_options, secrets):
                 r = build_root.run(
                     [f"/run/osbuild/lib/stages/{self.name}"],
                     binds=[f"{tree}:/run/osbuild/tree"],
@@ -212,13 +212,13 @@ class Pipeline:
                 finally:
                     subprocess.run(["umount", "--lazy", tmp], check=True)
 
-    def run(self, store, interactive=False, libdir=None, source_options=None):
+    def run(self, store, interactive=False, libdir=None, source_options=None, secrets=None):
         os.makedirs("/run/osbuild", exist_ok=True)
         object_store = objectstore.ObjectStore(store)
         results = {}
 
         if self.build:
-            r = self.build.run(store, interactive, libdir, source_options or {})
+            r = self.build.run(store, interactive, libdir, source_options, secrets)
             results["build"] = r
             if not r["success"]:
                 results["success"] = False
@@ -251,7 +251,8 @@ class Pipeline:
                                               build_tree,
                                               interactive=interactive,
                                               libdir=libdir,
-                                              source_options=source_options)
+                                              source_options=source_options,
+                                              secrets=secrets)
                                 results["stages"].append(r.as_dict())
                     except BuildError as err:
                         results["stages"].append(err.as_dict())

osbuild/sources.py

@@ -6,16 +6,18 @@ import threading
 
 
 class SourcesServer:
-    def __init__(self, socket_address, sources_dir, source_options):
+    def __init__(self, socket_address, sources_dir, source_options, secrets=None):
         self.socket_address = socket_address
         self.sources_dir = sources_dir
-        self.source_options = source_options
+        self.source_options = source_options or {}
+        self.secrets = secrets or {}
         self.event_loop = asyncio.new_event_loop()
         self.thread = threading.Thread(target=self._run_event_loop)
 
     def _run_source(self, source, checksums):
         msg = {
             "options": self.source_options.get(source, {}),
+            "secrets": self.secrets.get(source, {}),
             "checksums": checksums
         }