679b79c5e5
Import modules between files using the syntax `from . import foobar`, renaming what used to be `FooBar` to `foobar.FooBar` when moved to a separate file. In __init__.py only import what is meant to be public API. Signed-off-by: Tom Gundersen <teg@jklm.no>
251 lines
8.9 KiB
Python
251 lines
8.9 KiB
Python
|
|
import contextlib
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import tempfile
|
|
from . import buildroot
|
|
from . import objectstore
|
|
from . import remoteloop
|
|
|
|
|
|
RESET = "\033[0m"
|
|
BOLD = "\033[1m"
|
|
|
|
|
|
class StageFailed(Exception):
|
|
def __init__(self, name, returncode, output):
|
|
super(StageFailed, self).__init__()
|
|
self.name = name
|
|
self.returncode = returncode
|
|
self.output = output
|
|
|
|
|
|
class AssemblerFailed(Exception):
|
|
def __init__(self, name, returncode, output):
|
|
super(AssemblerFailed, self).__init__()
|
|
self.name = name
|
|
self.returncode = returncode
|
|
self.output = output
|
|
|
|
|
|
def print_header(title, options):
|
|
print()
|
|
print(f"{RESET}{BOLD}{title}{RESET} " + json.dumps(options or {}, indent=2))
|
|
print()
|
|
|
|
|
|
class Stage:
|
|
def __init__(self, name, build, base, options):
|
|
m = hashlib.sha256()
|
|
m.update(json.dumps(name, sort_keys=True).encode())
|
|
m.update(json.dumps(build, sort_keys=True).encode())
|
|
m.update(json.dumps(base, sort_keys=True).encode())
|
|
m.update(json.dumps(options, sort_keys=True).encode())
|
|
|
|
self.id = m.hexdigest()
|
|
self.name = name
|
|
self.options = options
|
|
|
|
def description(self):
|
|
description = {}
|
|
description["name"] = self.name
|
|
if self.options:
|
|
description["options"] = self.options
|
|
return description
|
|
|
|
def run(self, tree, build_tree, interactive=False, check=True, libdir=None):
|
|
with buildroot.BuildRoot(build_tree) as build_root:
|
|
if interactive:
|
|
print_header(f"{self.name}: {self.id}", self.options)
|
|
|
|
args = {
|
|
"tree": "/run/osbuild/tree",
|
|
"options": self.options,
|
|
}
|
|
|
|
path = "/run/osbuild/lib" if libdir else "/usr/libexec/osbuild"
|
|
r = build_root.run(
|
|
[f"{path}/osbuild-run", f"{path}/stages/{self.name}"],
|
|
binds=[f"{tree}:/run/osbuild/tree"],
|
|
readonly_binds=[f"{libdir}:{path}"] if libdir else [],
|
|
encoding="utf-8",
|
|
input=json.dumps(args),
|
|
stdout=None if interactive else subprocess.PIPE,
|
|
stderr=subprocess.STDOUT
|
|
)
|
|
if check and r.returncode != 0:
|
|
raise StageFailed(self.name, r.returncode, r.stdout)
|
|
|
|
return {
|
|
"name": self.name,
|
|
"returncode": r.returncode,
|
|
"output": r.stdout
|
|
}
|
|
|
|
|
|
class Assembler:
|
|
def __init__(self, name, options):
|
|
self.name = name
|
|
self.options = options
|
|
|
|
def description(self):
|
|
description = {}
|
|
description["name"] = self.name
|
|
if self.options:
|
|
description["options"] = self.options
|
|
return description
|
|
|
|
def run(self, tree, build_tree, output_dir=None, interactive=False, check=True, libdir=None):
|
|
with buildroot.BuildRoot(build_tree) as build_root:
|
|
if interactive:
|
|
print_header(f"Assembling: {self.name}", self.options)
|
|
|
|
args = {
|
|
"tree": "/run/osbuild/tree",
|
|
"options": self.options,
|
|
}
|
|
|
|
binds = []
|
|
if output_dir:
|
|
os.makedirs(output_dir, exist_ok=True)
|
|
binds.append(f"{output_dir}:/run/osbuild/output")
|
|
args["output_dir"] = "/run/osbuild/output"
|
|
|
|
path = "/run/osbuild/lib" if libdir else "/usr/libexec/osbuild"
|
|
with build_root.bound_socket("remoteloop") as sock, \
|
|
remoteloop.LoopServer(sock):
|
|
r = build_root.run(
|
|
[f"{path}/osbuild-run", f"{path}/assemblers/{self.name}"],
|
|
binds=binds,
|
|
readonly_binds=[f"{tree}:/run/osbuild/tree"] + ([f"{libdir}:{path}"] if libdir else []),
|
|
encoding="utf-8",
|
|
input=json.dumps(args),
|
|
stdout=None if interactive else subprocess.PIPE,
|
|
stderr=subprocess.STDOUT)
|
|
if check and r.returncode != 0:
|
|
raise AssemblerFailed(self.name, r.returncode, r.stdout)
|
|
|
|
return {
|
|
"name": self.name,
|
|
"returncode": r.returncode,
|
|
"output": r.stdout
|
|
}
|
|
|
|
|
|
class Pipeline:
|
|
def __init__(self, build=None):
|
|
self.build = build
|
|
self.stages = []
|
|
self.assembler = None
|
|
|
|
def get_id(self):
|
|
return self.stages[-1].id if self.stages else None
|
|
|
|
def add_stage(self, name, options=None):
|
|
build = self.build.get_id() if self.build else None
|
|
stage = Stage(name, build, self.get_id(), options or {})
|
|
self.stages.append(stage)
|
|
|
|
def set_assembler(self, name, options=None):
|
|
self.assembler = Assembler(name, options or {})
|
|
|
|
def description(self):
|
|
description = {}
|
|
if self.build:
|
|
description["build"] = self.build.description()
|
|
if self.stages:
|
|
description["stages"] = [s.description() for s in self.stages]
|
|
if self.assembler:
|
|
description["assembler"] = self.assembler.description()
|
|
return description
|
|
|
|
@contextlib.contextmanager
|
|
def get_buildtree(self, object_store):
|
|
if self.build:
|
|
with object_store.get_tree(self.build.get_id()) as tree:
|
|
yield tree
|
|
else:
|
|
with tempfile.TemporaryDirectory(dir=object_store.store) as tmp:
|
|
subprocess.run(["mount", "-o", "bind,ro,mode=0755", "/", tmp], check=True)
|
|
try:
|
|
yield tmp
|
|
finally:
|
|
subprocess.run(["umount", "--lazy", tmp], check=True)
|
|
|
|
def run(self, output_dir, store, interactive=False, check=True, libdir=None):
|
|
os.makedirs("/run/osbuild", exist_ok=True)
|
|
object_store = objectstore.ObjectStore(store)
|
|
results = {
|
|
"stages": []
|
|
}
|
|
if self.build:
|
|
r = self.build.run(None, store, interactive, check, libdir)
|
|
results["build"] = r
|
|
if r["returncode"] != 0:
|
|
results["returncode"] = r["returncode"]
|
|
return results
|
|
|
|
with self.get_buildtree(object_store) as build_tree:
|
|
if self.stages:
|
|
if not object_store.has_tree(self.get_id()):
|
|
# Find the last stage that already exists in the object store, and use
|
|
# that as the base.
|
|
base = None
|
|
base_idx = -1
|
|
for i in range(len(self.stages) - 1, 0, -1):
|
|
if object_store.has_tree(self.stages[i].id):
|
|
base = self.stages[i].id
|
|
base_idx = i
|
|
break
|
|
# The tree does not exist. Create it and save it to the object store. If
|
|
# two run() calls race each-other, two trees may be generated, and it
|
|
# is nondeterministic which of them will end up referenced by the tree_id
|
|
# in the content store. However, we guarantee that all tree_id's and all
|
|
# generated trees remain valid.
|
|
with object_store.new_tree(self.get_id(), base_id=base) as tree:
|
|
for stage in self.stages[base_idx + 1:]:
|
|
r = stage.run(tree,
|
|
build_tree,
|
|
interactive=interactive,
|
|
check=check,
|
|
libdir=libdir)
|
|
results["stages"].append(r)
|
|
if r["returncode"] != 0:
|
|
results["returncode"] = r["returncode"]
|
|
return results
|
|
|
|
if self.assembler:
|
|
with object_store.get_tree(self.get_id()) as tree:
|
|
r = self.assembler.run(tree,
|
|
build_tree,
|
|
output_dir=output_dir,
|
|
interactive=interactive,
|
|
check=check,
|
|
libdir=libdir)
|
|
results["assembler"] = r
|
|
if r["returncode"] != 0:
|
|
results["returncode"] = r["returncode"]
|
|
return results
|
|
|
|
results["returncode"] = 0
|
|
return results
|
|
|
|
|
|
def load(description):
|
|
build_description = description.get("build")
|
|
if build_description:
|
|
build = load(build_description)
|
|
else:
|
|
build = None
|
|
pipeline = Pipeline(build)
|
|
|
|
for s in description.get("stages", []):
|
|
pipeline.add_stage(s["name"], s.get("options", {}))
|
|
|
|
a = description.get("assembler")
|
|
if a:
|
|
pipeline.set_assembler(a["name"], a.get("options", {}))
|
|
|
|
return pipeline
|