From 29f2a68eebadc93292eaed3b7e06ff1186b27e49 Mon Sep 17 00:00:00 2001
From: Christian Kellner <christian@kellner.me>
Date: Mon, 25 Oct 2021 16:03:04 +0200
Subject: [PATCH] osbuild: on-demand building of pipelines

Use the new Manifest.depsolve function to only build the pipelines that
were explicitly requested and their dependencies, taking into account
what is already present in the store.
Since now not all pipeline will be built, there wont be a result entry
for all the pipelines, thus the format version 2 result formatting was
changed to not require the pipeline to be present in result set.
---
 osbuild/formats/v2.py   |  2 +-
 osbuild/main_cli.py     | 14 +++++++++-----
 osbuild/pipeline.py     |  4 ++--
 test/mod/test_fmt_v1.py |  2 +-
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/osbuild/formats/v2.py b/osbuild/formats/v2.py
index c3b32af0..4b168b39 100644
--- a/osbuild/formats/v2.py
+++ b/osbuild/formats/v2.py
@@ -405,7 +405,7 @@ def output(manifest: Manifest, res: Dict) -> Dict:
         # gather all the metadata
         for p in manifest.pipelines.values():
             data = {}
-            r = res[p.id]
+            r = res.get(p.id, {})
             for stage in r.get("stages", []):
                 md = stage.get("metadata")
                 if not md:
diff --git a/osbuild/main_cli.py b/osbuild/main_cli.py
index 81246535..469a9a8a 100644
--- a/osbuild/main_cli.py
+++ b/osbuild/main_cli.py
@@ -84,7 +84,7 @@ def parse_arguments(sys_argv):
     return parser.parse_args(sys_argv[1:])
 
 
-# pylint: disable=too-many-branches,too-many-return-statements
+# pylint: disable=too-many-branches,too-many-return-statements,too-many-statements
 def osbuild_cli():
     args = parse_arguments(sys.argv)
     desc = parse_manifest(args.manifest_path)
@@ -110,7 +110,8 @@ def osbuild_cli():
 
     manifest = fmt.load(desc, index)
 
-    unresolved = [e for e in args.export if e not in manifest]
+    exports = set(args.export)
+    unresolved = [e for e in exports if e not in manifest]
     if unresolved:
         for name in unresolved:
             print(f"Export {BOLD}{name}{RESET} not found!")
@@ -133,7 +134,7 @@ def osbuild_cli():
 
     output_directory = args.output_directory
 
-    if args.export and not output_directory:
+    if exports and not output_directory:
         print("Need --output-directory for --export")
         return 1
 
@@ -147,16 +148,19 @@ def osbuild_cli():
     try:
         with ObjectStore(args.store) as object_store:
 
+            pipelines = manifest.depsolve(object_store, exports)
+
             manifest.download(object_store, monitor, args.libdir)
 
             r = manifest.build(
                 object_store,
+                pipelines,
                 monitor,
                 args.libdir
             )
 
-            if r["success"] and args.export:
-                for pid in args.export:
+            if r["success"] and exports:
+                for pid in exports:
                     export(pid, output_directory, object_store, manifest)
 
     except KeyboardInterrupt:
diff --git a/osbuild/pipeline.py b/osbuild/pipeline.py
index abe5398f..99e4b7db 100644
--- a/osbuild/pipeline.py
+++ b/osbuild/pipeline.py
@@ -405,10 +405,10 @@ class Manifest:
 
         return list(map(lambda x: x.name, reversed(build.values())))
 
-    def build(self, store, monitor, libdir):
+    def build(self, store, pipelines, monitor, libdir):
         results = {"success": True}
 
-        for pl in self.pipelines.values():
+        for pl in map(self.get, pipelines):
             res = pl.run(store, monitor, libdir)
             results[pl.id] = res
             if not res["success"]:
diff --git a/test/mod/test_fmt_v1.py b/test/mod/test_fmt_v1.py
index 4f786f53..ef0104ca 100644
--- a/test/mod/test_fmt_v1.py
+++ b/test/mod/test_fmt_v1.py
@@ -82,7 +82,7 @@ class TestFormatV1(unittest.TestCase):
         libdir = os.path.abspath(os.curdir)
         store = ObjectStore(storedir)
 
-        res = manifest.build(store, monitor, libdir)
+        res = manifest.build(store, manifest.pipelines, monitor, libdir)
         return res
 
     def test_canonical(self):