osbuild.py/pipeline: add the concept of a content store

Whenever an assembler is not specified, the output tree is instead
saved to the content store, in a directory named after the pipeline
id.

This should render the io.weldr.tree assembler redundant.

In order to build the samples as before, specify the content store
as the input directory to build any pipeline that uses the
io.weldr.untree stage.

Signed-off-by: Tom Gundersen <teg@jklm.no>

osbuild

@@ -17,6 +17,9 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Build operating system images")
     parser.add_argument("pipeline_path", metavar="PIPELINE",
                         help="json file containing the pipeline that should be built")
+    parser.add_argument("--objects", metavar="DIRECTORY", type=os.path.abspath,
+                        default=".osbuild/objects",
+                        help="the directory where intermediary os trees are stored")
     requiredNamed = parser.add_argument_group('required named arguments')
     requiredNamed.add_argument("-i", "--input", dest="input_dir", metavar="DIRECTORY", type=os.path.abspath,
                         help="provide the contents of DIRECTORY to the first stage", required=True)
@@ -28,7 +31,7 @@ if __name__ == "__main__":
 
     with open(args.pipeline_path) as f:
         pipeline = json.load(f)
-    pipeline = osbuild.Pipeline(pipeline)
+    pipeline = osbuild.Pipeline(pipeline, args.objects)
 
     print()
     print(f"{RESET}{BOLD}Pipeline: {pipeline.id}{RESET}")

osbuild.py

@@ -2,6 +2,7 @@
 import hashlib
 import json
 import os
+import shutil
 import subprocess
 import sys
 import tempfile
@@ -203,13 +204,16 @@ def print_header(title, options, machine_name):
     print()
 
 class Pipeline:
-    def __init__(self, pipeline):
+    def __init__(self, pipeline, objects):
         m = hashlib.sha256()
         m.update(json.dumps(pipeline, sort_keys=True).encode())
 
         self.id = m.hexdigest()
         self.stages = pipeline["stages"]
         self.assembler = pipeline.get("assembler")
+        self.objects = objects
+
+        os.makedirs(objects, exist_ok=True)
 
     def run(self, input_dir, output_dir, interactive=False):
         results = {
@@ -231,5 +235,12 @@ class Pipeline:
                     print_header(f"Assembling: {name}", options, buildroot.machine_name)
                 r = buildroot.run_assembler(self.assembler, tree, input_dir, output_dir, interactive)
                 results["assembler"] = r
+            else:
+                output_tree = os.path.join(self.objects, self.id)
+
+                shutil.rmtree(output_tree, ignore_errors=True)
+                os.makedirs(output_tree, mode=0o755)
+
+                subprocess.run(["cp", "-a", f"{tree}/.", output_tree], check=True)
 
         self.results = results

samples/base-qcow2.json

@@ -4,7 +4,7 @@
     {
       "name": "io.weldr.untree",
       "options": {
-        "tree": "base-with-grub2"
+        "tree": "efc7b094c174d9f5fd66c863db7128415f5df0f073125716f5279554f13a659f"
       }
     }
   ],

samples/base-targz.json

@@ -4,7 +4,7 @@
     {
       "name": "io.weldr.untree",
       "options": {
-        "tree": "base"
+        "tree": "1f663f817473ffa5b01241b17adbd71bc734962313f5d4eef230073c0ac5884e"
       }
     }
   ],

samples/base-with-grub2.json

@@ -4,7 +4,7 @@
     {
       "name": "io.weldr.untree",
       "options": {
-        "tree": "base"
+        "tree": "1f663f817473ffa5b01241b17adbd71bc734962313f5d4eef230073c0ac5884e"
       }
     },
     {
@@ -15,11 +15,5 @@
 	"partition_table_id": "0xdeadbeef"
       }
     }
-  ],
+  ]
-  "assembler": {
-    "name": "io.weldr.tree",
-    "options": {
-      "tree": "base-with-grub2"
-    }
-  }
 }

samples/base-with-locale.json

@@ -2,9 +2,9 @@
   "name": "base-with-locale",
   "stages": [
     {
-      "name": "io.weldr.untargz",
+      "name": "io.weldr.untree",
       "options": {
-        "filename": "base.tar.gz"
+        "tree": "1f663f817473ffa5b01241b17adbd71bc734962313f5d4eef230073c0ac5884e"
       }
     },
     {
@@ -13,11 +13,5 @@
         "language": "en_US"
       }
     }
-  ],
+  ]
-  "assembler": {
-    "name": "io.weldr.tree",
-    "options": {
-      "tree": "base-with-locale"
-    }
-  }
 }

samples/base-with-selinux.json

@@ -4,7 +4,7 @@
     {
       "name": "io.weldr.untree",
       "options": {
-        "tree": "base"
+        "tree": "1f663f817473ffa5b01241b17adbd71bc734962313f5d4eef230073c0ac5884e"
       }
     },
     {
@@ -13,11 +13,5 @@
         "file_contexts": "etc/selinux/targeted/contexts/files/file_contexts"
       }
     }
-  ],
+  ]
-  "assembler": {
-    "name": "io.weldr.tree",
-    "options": {
-      "tree": "base-with-selinux"
-    }
-  }
 }

samples/base.json

@@ -20,12 +20,5 @@
         ]
       }
     }
-  ],
+  ]
-  "assembler":
-    {
-      "name": "io.weldr.tree",
-      "options": {
-        "tree": "base"
-      }
-    }
 }