From 65151e22ff3fffc3827de5cc90014a1dbc93aab4 Mon Sep 17 00:00:00 2001 From: Tom Gundersen Date: Tue, 9 Jul 2019 10:28:37 +0200 Subject: [PATCH] osbuild.py: assign ids to stages rather than to pipelines Compute a hash based on the content of a stage, together with the hash of its parent stage. The output of a pipeline is saved by the id of the last stage. This is largely equivalent to the current logic, where it is the pipeline that contains the id, but this means that the ids are indepedent of how pipelines are split, the only thing that matters is the sequence of stages, not whether or not they are in one or several interdependent pipelines. Signed-off-by: Tom Gundersen --- osbuild | 3 --- osbuild.py | 26 ++++++++++++++++---------- samples/base-qcow2.json | 2 +- samples/base-with-grub2.json | 2 +- samples/base-with-locale.json | 2 +- samples/base-with-selinux.json | 2 +- 6 files changed, 20 insertions(+), 17 deletions(-) diff --git a/osbuild b/osbuild index bc29c1d6..c998f3e5 100755 --- a/osbuild +++ b/osbuild @@ -31,9 +31,6 @@ if __name__ == "__main__": pipeline = json.load(f) pipeline = osbuild.Pipeline(pipeline, args.objects) - print() - print(f"{RESET}{BOLD}Pipeline: {pipeline.id}{RESET}") - try: pipeline.run(args.output_dir, interactive=True) except KeyboardInterrupt: diff --git a/osbuild.py b/osbuild.py index 2a577198..d816cd10 100644 --- a/osbuild.py +++ b/osbuild.py @@ -148,7 +148,14 @@ def _get_system_resources_from_etc(resources): class Stage: - def __init__(self, name, options, resources): + def __init__(self, name, base, options, resources): + m = hashlib.sha256() + m.update(json.dumps(name, sort_keys=True).encode()) + m.update(json.dumps(base, sort_keys=True).encode()) + m.update(json.dumps(options, sort_keys=True).encode()) + m.update(json.dumps(resources, sort_keys=True).encode()) + + self.id = m.hexdigest() self.name = name self.options = options self.resources = resources @@ -156,7 +163,7 @@ class Stage: def run(self, tree, interactive=False): with BuildRoot() as buildroot: if interactive: - print_header(f"{self.name}", self.options, buildroot.machine_name) + print_header(f"{self.name}: {self.id}", self.options, buildroot.machine_name) args = { "tree": "/run/osbuild/tree", @@ -205,10 +212,6 @@ class Assembler: class Pipeline: def __init__(self, pipeline, objects): - m = hashlib.sha256() - m.update(json.dumps(pipeline, sort_keys=True).encode()) - - self.id = m.hexdigest() self.base = pipeline.get("base") self.stages = pipeline.get("stages", []) self.assembler = pipeline.get("assembler") @@ -221,17 +224,20 @@ class Pipeline: "stages": [] } with tmpfs() as tree: - if self.base: - input_tree = os.path.join(self.objects, self.base) + base = self.base + + if base: + input_tree = os.path.join(self.objects, base) subprocess.run(["cp", "-a", f"{input_tree}/.", tree], check=True) for stage in self.stages: name = stage["name"] options = stage.get("options", {}) resources = stage.get("systemResourcesFromEtc", []) - stage = Stage(name, options, resources) + stage = Stage(name, base, options, resources) r = stage.run(tree, interactive) results["stages"].append(r) + base = stage.id if self.assembler: name = self.assembler["name"] @@ -241,7 +247,7 @@ class Pipeline: r = assembler.run(tree, output_dir, interactive) results["assembler"] = r else: - output_tree = os.path.join(self.objects, self.id) + output_tree = os.path.join(self.objects, base) shutil.rmtree(output_tree, ignore_errors=True) os.makedirs(output_tree, mode=0o755) subprocess.run(["cp", "-a", f"{tree}/.", output_tree], check=True) diff --git a/samples/base-qcow2.json b/samples/base-qcow2.json index 161ed603..d7de55d8 100644 --- a/samples/base-qcow2.json +++ b/samples/base-qcow2.json @@ -1,6 +1,6 @@ { "name": "base-qcow2", - "base": "f571638c0a21a7141e2391d476433c1679831e4f21df6555c2ef53c4446b7603", + "base": "445c09ba71fd656aadcc2c1e84adb12dcc5fd193fcbdc386b63e2fc02e134e98", "assembler": { "name": "io.weldr.qcow2", diff --git a/samples/base-with-grub2.json b/samples/base-with-grub2.json index 7a898bc7..a9375b7e 100644 --- a/samples/base-with-grub2.json +++ b/samples/base-with-grub2.json @@ -1,6 +1,6 @@ { "name": "base-with-grub2", - "base": "1f663f817473ffa5b01241b17adbd71bc734962313f5d4eef230073c0ac5884e", + "base": "baa7c544f69bb82d5f3992dd534e203a8f8442a47e751a286481f391a2d1f075", "stages": [ { "name": "io.weldr.grub2", diff --git a/samples/base-with-locale.json b/samples/base-with-locale.json index fb1bb261..7181847b 100644 --- a/samples/base-with-locale.json +++ b/samples/base-with-locale.json @@ -1,6 +1,6 @@ { "name": "base-with-locale", - "base": "1f663f817473ffa5b01241b17adbd71bc734962313f5d4eef230073c0ac5884e", + "base": "baa7c544f69bb82d5f3992dd534e203a8f8442a47e751a286481f391a2d1f075", "stages": [ { "name": "io.weldr.locale", diff --git a/samples/base-with-selinux.json b/samples/base-with-selinux.json index f72dcac0..2fa7ac15 100644 --- a/samples/base-with-selinux.json +++ b/samples/base-with-selinux.json @@ -1,6 +1,6 @@ { "name": "base-with-selinux", - "base": "1f663f817473ffa5b01241b17adbd71bc734962313f5d4eef230073c0ac5884e", + "base": "baa7c544f69bb82d5f3992dd534e203a8f8442a47e751a286481f391a2d1f075", "stages": [ { "name": "io.weldr.selinux",