osbuild: on-demand building of pipelines

Use the new Manifest.depsolve function to only build the pipelines that
were explicitly requested and their dependencies, taking into account
what is already present in the store.
Since now not all pipeline will be built, there wont be a result entry
for all the pipelines, thus the format version 2 result formatting was
changed to not require the pipeline to be present in result set.

osbuild/formats/v2.py

@@ -405,7 +405,7 @@ def output(manifest: Manifest, res: Dict) -> Dict:
         # gather all the metadata
         for p in manifest.pipelines.values():
             data = {}
-            r = res[p.id]
+            r = res.get(p.id, {})
             for stage in r.get("stages", []):
                 md = stage.get("metadata")
                 if not md:

osbuild/main_cli.py

@@ -84,7 +84,7 @@ def parse_arguments(sys_argv):
     return parser.parse_args(sys_argv[1:])
 
 
-# pylint: disable=too-many-branches,too-many-return-statements
+# pylint: disable=too-many-branches,too-many-return-statements,too-many-statements
 def osbuild_cli():
     args = parse_arguments(sys.argv)
     desc = parse_manifest(args.manifest_path)
@@ -110,7 +110,8 @@ def osbuild_cli():
 
     manifest = fmt.load(desc, index)
 
-    unresolved = [e for e in args.export if e not in manifest]
+    exports = set(args.export)
+    unresolved = [e for e in exports if e not in manifest]
     if unresolved:
         for name in unresolved:
             print(f"Export {BOLD}{name}{RESET} not found!")
@@ -133,7 +134,7 @@ def osbuild_cli():
 
     output_directory = args.output_directory
 
-    if args.export and not output_directory:
+    if exports and not output_directory:
         print("Need --output-directory for --export")
         return 1
 
@@ -147,16 +148,19 @@ def osbuild_cli():
     try:
         with ObjectStore(args.store) as object_store:
 
+            pipelines = manifest.depsolve(object_store, exports)
+
             manifest.download(object_store, monitor, args.libdir)
 
             r = manifest.build(
                 object_store,
+                pipelines,
                 monitor,
                 args.libdir
             )
 
-            if r["success"] and args.export:
+            if r["success"] and exports:
-                for pid in args.export:
+                for pid in exports:
                     export(pid, output_directory, object_store, manifest)
 
     except KeyboardInterrupt:

osbuild/pipeline.py

@@ -405,10 +405,10 @@ class Manifest:
 
         return list(map(lambda x: x.name, reversed(build.values())))
 
-    def build(self, store, monitor, libdir):
+    def build(self, store, pipelines, monitor, libdir):
         results = {"success": True}
 
-        for pl in self.pipelines.values():
+        for pl in map(self.get, pipelines):
             res = pl.run(store, monitor, libdir)
             results[pl.id] = res
             if not res["success"]:

test/mod/test_fmt_v1.py

@@ -82,7 +82,7 @@ class TestFormatV1(unittest.TestCase):
         libdir = os.path.abspath(os.curdir)
         store = ObjectStore(storedir)
 
-        res = manifest.build(store, monitor, libdir)
+        res = manifest.build(store, manifest.pipelines, monitor, libdir)
         return res
 
     def test_canonical(self):