1da71ebbb4
Introduce support for ppc64le (Open Firmware). The main difference to x86 legacy, i.e. non-efi, is that no stage 1 is required because the core image is stored on a special 'PReP' partition, which must be marked as bootable. The firmware then looks for that partition and directly loads the core from there and executes it. Introduce a `platform` parameter for the grub installer code which controls various platform depended aspects, including a) the path for the modules, b) what modules are compiled into the core, c) if the boot image is written to the MBR and 4) where to write the core image, i.e. mbr-gap or PReP partition.
538 lines
18 KiB
Python
Executable file
538 lines
18 KiB
Python
Executable file
#!/usr/bin/python3
|
|
|
|
import contextlib
|
|
import json
|
|
import os
|
|
import socket
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from typing import List, BinaryIO
|
|
import osbuild.remoteloop as remoteloop
|
|
|
|
STAGE_DESC = "Assemble a bootable partitioned disk image with qemu-img"
|
|
STAGE_INFO = """
|
|
Assemble a bootable partitioned disk image using `qemu-img`.
|
|
|
|
Creates a sparse partitioned disk image of type `pttype` of a given `size`,
|
|
with a partition table according to `partitions` or a MBR partitioned disk
|
|
having a single bootable partition containing the root filesystem if the
|
|
`pttype` property is absent.
|
|
|
|
If the partition type is MBR it installs GRUB2 (using the buildhost's
|
|
`/usr/lib/grub/i386-pc/boot.img` etc.) as the bootloader.
|
|
|
|
Copies the tree contents into the root filesystem and then converts the raw
|
|
sparse image into the format requested with the `fmt` option.
|
|
|
|
Buildhost commands used: `truncate`, `mount`, `umount`, `sfdisk`,
|
|
`grub2-mkimage`, `mkfs.ext4` or `mkfs.xfs`, `qemu-img`.
|
|
"""
|
|
STAGE_OPTS = """
|
|
"required": ["format", "filename", "ptuuid", "size"],
|
|
"oneOf": [{
|
|
"required": ["root_fs_uuid"]
|
|
},{
|
|
"required": ["pttype", "partitions"]
|
|
}],
|
|
"properties": {
|
|
"format": {
|
|
"description": "Image file format to use",
|
|
"type": "string",
|
|
"enum": ["raw", "qcow2", "vdi", "vmdk", "vpc"]
|
|
},
|
|
"filename": {
|
|
"description": "Image filename",
|
|
"type": "string"
|
|
},
|
|
"partitions": {
|
|
"description": "Partition layout ",
|
|
"type": "array",
|
|
"items": {
|
|
"description": "Description of one partition",
|
|
"type": "object",
|
|
"properties": {
|
|
"bootable": {
|
|
"description": "Mark the partition as bootable (dos)",
|
|
"type": "boolean"
|
|
},
|
|
"name": {
|
|
"description": "The partition name (GPT)",
|
|
"type": "string"
|
|
},
|
|
"size": {
|
|
"description": "The size of this partition",
|
|
"type": "integer"
|
|
},
|
|
"start": {
|
|
"description": "The start offset of this partition",
|
|
"type": "integer"
|
|
},
|
|
"type": {
|
|
"description": "The partition type (UUID or identifier)",
|
|
"type": "string"
|
|
},
|
|
"filesystem": {
|
|
"description": "Description of the filesystem",
|
|
"type": "object",
|
|
"required": ["mountpoint", "type", "uuid"],
|
|
"properties": {
|
|
"label": {
|
|
"description": "Label for the filesystem",
|
|
"type": "string"
|
|
},
|
|
"mountpoint": {
|
|
"description": "Where to mount the partition",
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"description": "Type of the filesystem",
|
|
"type": "string",
|
|
"enum": ["ext4", "xfs", "vfat"]
|
|
},
|
|
"uuid": {
|
|
"description": "UUID for the filesystem",
|
|
"type": "string"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"ptuuid": {
|
|
"description": "UUID for the disk image's partition table",
|
|
"type": "string"
|
|
},
|
|
"pttype": {
|
|
"description": "The type of the partition table",
|
|
"type": "string",
|
|
"enum": ["mbr", "gpt"]
|
|
},
|
|
"root_fs_uuid": {
|
|
"description": "UUID for the root filesystem",
|
|
"type": "string"
|
|
},
|
|
"size": {
|
|
"description": "Virtual disk size",
|
|
"type": "string"
|
|
},
|
|
"root_fs_type": {
|
|
"description": "Type of the root filesystem",
|
|
"type": "string",
|
|
"enum": ["ext4", "xfs"],
|
|
"default": "ext4"
|
|
}
|
|
}
|
|
"""
|
|
|
|
@contextlib.contextmanager
|
|
def mount(source, dest):
|
|
subprocess.run(["mount", source, dest], check=True)
|
|
try:
|
|
yield dest
|
|
finally:
|
|
subprocess.run(["umount", "-R", dest], check=True)
|
|
|
|
|
|
def mkfs_ext4(device, uuid, label):
|
|
opts = []
|
|
if label:
|
|
opts = ["-L", label]
|
|
subprocess.run(["mkfs.ext4", "-U", uuid] + opts + [device],
|
|
input="y", encoding='utf-8', check=True)
|
|
|
|
|
|
def mkfs_xfs(device, uuid, label):
|
|
opts = []
|
|
if label:
|
|
opts = ["-L", label]
|
|
subprocess.run(["mkfs.xfs", "-m", f"uuid={uuid}"] + opts + [device],
|
|
encoding='utf-8', check=True)
|
|
|
|
|
|
def mkfs_vfat(device, uuid, label):
|
|
volid = uuid.replace('-', '')
|
|
opts = []
|
|
if label:
|
|
opts = ["-n", label]
|
|
subprocess.run(["mkfs.vfat", "-i", volid] + opts + [device], encoding='utf-8', check=True)
|
|
|
|
|
|
class Filesystem:
|
|
def __init__(self,
|
|
fstype: str,
|
|
uuid: str,
|
|
mountpoint: str,
|
|
label: str = None):
|
|
self.type = fstype
|
|
self.uuid = uuid
|
|
self.mountpoint = mountpoint
|
|
self.label = label
|
|
|
|
def make_at(self, device: str):
|
|
fs_type = self.type
|
|
if fs_type == "ext4":
|
|
maker = mkfs_ext4
|
|
elif fs_type == "xfs":
|
|
maker = mkfs_xfs
|
|
elif fs_type == "vfat":
|
|
maker = mkfs_vfat
|
|
else:
|
|
raise ValueError(f"Unknown filesystem type '{fs_type}'")
|
|
maker(device, self.uuid, self.label)
|
|
|
|
|
|
class Partition:
|
|
def __init__(self,
|
|
pttype: str = None,
|
|
start: int = None,
|
|
size: int = None,
|
|
bootable: bool = False,
|
|
name: str = None,
|
|
filesystem: Filesystem = None):
|
|
self.type = pttype
|
|
self.start = start
|
|
self.size = size
|
|
self.bootable = bootable
|
|
self.name = name
|
|
self.filesystem = filesystem
|
|
self.index = None
|
|
|
|
@property
|
|
def start_in_bytes(self):
|
|
return (self.start or 0) * 512
|
|
|
|
@property
|
|
def size_in_bytes(self):
|
|
return (self.size or 0) * 512
|
|
|
|
@property
|
|
def mountpoint(self):
|
|
if self.filesystem is None:
|
|
return None
|
|
return self.filesystem.mountpoint
|
|
|
|
@property
|
|
def fs_type(self):
|
|
if self.filesystem is None:
|
|
return None
|
|
return self.filesystem.type
|
|
|
|
@property
|
|
def fs_uuid(self):
|
|
if self.filesystem is None:
|
|
return None
|
|
return self.filesystem.uuid
|
|
|
|
|
|
class PartitionTable:
|
|
def __init__(self, label, uuid, partitions):
|
|
self.label = label
|
|
self.uuid = uuid
|
|
self.partitions = partitions or []
|
|
|
|
def __getitem__(self, key) -> Partition:
|
|
return self.partitions[key]
|
|
|
|
def partitions_with_filesystems(self) -> List[Partition]:
|
|
"""Return partitions with filesystems sorted by hierarchy"""
|
|
def mountpoint_len(p):
|
|
return len(p.mountpoint)
|
|
parts_fs = filter(lambda p: p.filesystem is not None, self.partitions)
|
|
return sorted(parts_fs, key=mountpoint_len)
|
|
|
|
def partition_containing_root(self) -> Partition:
|
|
"""Return the partition containing the root filesystem"""
|
|
for p in self.partitions:
|
|
if p.mountpoint and p.mountpoint == "/":
|
|
return p
|
|
return None
|
|
|
|
def partition_containing_boot(self) -> Partition:
|
|
"""Return the partition containing /boot"""
|
|
for p in self.partitions_with_filesystems():
|
|
if p.mountpoint == "/boot":
|
|
return p
|
|
# fallback to the root partition
|
|
return self.partition_containing_root()
|
|
|
|
def find_prep_partition(self) -> Partition:
|
|
"""Find the PReP partition'"""
|
|
if self.label == "dos":
|
|
prep_type = "41"
|
|
elif self.label == "gpt":
|
|
prep_type = "9E1A2D38-C612-4316-AA26-8B49521E5A8B"
|
|
|
|
for part in self.partitions:
|
|
if part.type.upper() == prep_type:
|
|
return part
|
|
return None
|
|
|
|
def write_to(self, target, sync=True):
|
|
"""Write the partition table to disk"""
|
|
# generate the command for sfdisk to create the table
|
|
command = f"label: {self.label}\nlabel-id: {self.uuid}"
|
|
for partition in self.partitions:
|
|
fields = []
|
|
for field in ["start", "size", "type", "name"]:
|
|
value = getattr(partition, field)
|
|
if value:
|
|
fields += [f'{field}="{value}"']
|
|
if partition.bootable:
|
|
fields += ["bootable"]
|
|
command += "\n" + ", ".join(fields)
|
|
|
|
subprocess.run(["sfdisk", "-q", target],
|
|
input=command,
|
|
encoding='utf-8',
|
|
check=True)
|
|
|
|
if sync:
|
|
self.update_from(target)
|
|
|
|
def update_from(self, target):
|
|
"""Update and fill in missing information from disk"""
|
|
r = subprocess.run(["sfdisk", "--json", target],
|
|
stdout=subprocess.PIPE,
|
|
encoding='utf-8',
|
|
check=True)
|
|
disk_table = json.loads(r.stdout)["partitiontable"]
|
|
disk_parts = disk_table["partitions"]
|
|
|
|
assert len(disk_parts) == len(self.partitions)
|
|
for i, part in enumerate(self.partitions):
|
|
part.index = i
|
|
part.start = disk_parts[i]["start"]
|
|
part.size = disk_parts[i]["size"]
|
|
part.type = disk_parts[i].get("type")
|
|
part.name = disk_parts[i].get("name")
|
|
|
|
|
|
def filesystem_from_json(js) -> Filesystem:
|
|
return Filesystem(js["type"], js["uuid"], js["mountpoint"], js.get("label"))
|
|
|
|
|
|
def partition_from_json(js) -> Partition:
|
|
p = Partition(pttype=js.get("type"),
|
|
start=js.get("start"),
|
|
size=js.get("size"),
|
|
bootable=js.get("bootable"),
|
|
name=js.get("name"))
|
|
fs = js.get("filesystem")
|
|
if fs:
|
|
p.filesystem = filesystem_from_json(fs)
|
|
return p
|
|
|
|
|
|
def partition_table_from_options(options) -> PartitionTable:
|
|
ptuuid = options["ptuuid"]
|
|
pttype = options.get("pttype", "dos")
|
|
partitions = options.get("partitions")
|
|
|
|
if pttype == "mbr":
|
|
pttype = "dos"
|
|
|
|
if partitions is None:
|
|
# legacy mode, create a correct
|
|
root_fs_uuid = options["root_fs_uuid"]
|
|
root_fs_type = options.get("root_fs_type", "ext4")
|
|
partitions = [{
|
|
"bootable": True,
|
|
"type": "83",
|
|
"filesystem": {
|
|
"type": root_fs_type,
|
|
"uuid": root_fs_uuid,
|
|
"mountpoint": "/"
|
|
}
|
|
}]
|
|
parts = [partition_from_json(p) for p in partitions]
|
|
return PartitionTable(pttype, ptuuid, parts)
|
|
|
|
|
|
def grub2_write_core_mbrgap(core_f: BinaryIO,
|
|
image_f: BinaryIO,
|
|
pt: PartitionTable):
|
|
"""Write the core into the MBR gap"""
|
|
# For historic and performance reasons the first partition
|
|
# is aligned to a specific sector number (used to be 64,
|
|
# now it is 2048), which leaves a gap between it and the MBR,
|
|
# where the core image can be embedded in; also check it fits
|
|
core_size = os.fstat(core_f.fileno()).st_size
|
|
partition_offset = pt[0].start_in_bytes
|
|
assert core_size < partition_offset - 512
|
|
image_f.seek(512)
|
|
shutil.copyfileobj(core_f, image_f)
|
|
|
|
|
|
def grub2_write_core_prep_part(core_f: BinaryIO,
|
|
image_f: BinaryIO,
|
|
pt: PartitionTable):
|
|
"""Write the core to the prep partition"""
|
|
# On ppc64le with Open Firmware a special partition called
|
|
# 'PrEP partition' is used the store the grub2 core; the
|
|
# firmware looks for this partition and directly loads and
|
|
# executes the core form it.
|
|
prep_part = pt.find_prep_partition()
|
|
if prep_part is None:
|
|
raise ValueError("PrEP partition missing")
|
|
|
|
core_size = os.fstat(core_f.fileno()).st_size
|
|
assert core_size < prep_part.size_in_bytes - 512
|
|
image_f.seek(prep_part.start_in_bytes)
|
|
shutil.copyfileobj(core_f, image_f)
|
|
|
|
|
|
def install_grub2(image: str, pt: PartitionTable, options):
|
|
"""Install grub2 to image"""
|
|
platform = options.get("platform", "i386-pc")
|
|
|
|
boot_path = f"/usr/lib/grub/{platform}/boot.img"
|
|
core_path = "/var/tmp/grub2-core.img"
|
|
|
|
# Create the level-2 & 3 stages of the bootloader, aka the core
|
|
# it consists of the kernel plus the core modules required to
|
|
# to locate and load the rest of the grub modules, specifically
|
|
# the "normal.mod" (Stage 4) module.
|
|
# The exact list of modules required to be built into the core
|
|
# depends on the system: it is the minimal set needed to find
|
|
# read the partition and its filesystem containing said modules
|
|
# and the grub configuration [NB: efi systems work differently]
|
|
|
|
# find the partition containing /boot/grub2
|
|
boot_part = pt.partition_containing_boot()
|
|
|
|
# modules: access the disk and read the partition table:
|
|
# on x86 'biosdisk' is used to access the disk, on ppc64le
|
|
# with "Open Firmware" the latter is directly loading core
|
|
if platform == "i386-pc":
|
|
modules = ["biosdisk"]
|
|
else:
|
|
modules = []
|
|
|
|
modules += ["part_msdos"]
|
|
|
|
# modules: grubs needs to access the filesystems of /boot/grub2
|
|
fs_type = boot_part.fs_type or "unknown"
|
|
|
|
if fs_type == "ext4":
|
|
modules += ["ext2"]
|
|
elif fs_type == "xfs":
|
|
modules += ["xfs"]
|
|
else:
|
|
raise ValueError(f"unknown boot filesystem type: '{fs_type}'")
|
|
|
|
# identify the partition containing boot for grub2
|
|
if pt.label == "dos":
|
|
partid = "msdos" + str(boot_part.index + 1)
|
|
else:
|
|
raise ValueError(f"unsupported partition type: '{pt.label}'")
|
|
|
|
# now created the core image
|
|
subprocess.run(["grub2-mkimage",
|
|
"--verbose",
|
|
"--directory", f"/usr/lib/grub/{platform}",
|
|
"--prefix", f"(,{partid})/boot/grub2",
|
|
"--format", platform,
|
|
"--compression", "auto",
|
|
"--output", core_path] +
|
|
modules,
|
|
check=True)
|
|
|
|
with open(image, "rb+") as image_f:
|
|
if platform == "i386-pc":
|
|
# On x86, install the level-1 bootloader into the start of the MBR
|
|
# The purpose of this is simply to jump into the stage-2 bootloader.
|
|
# On ppc64le & Open Firmware stage-2 is loaded by the firmware
|
|
with open(boot_path, "rb") as boot_f:
|
|
# The boot.img file is 512 bytes, but we must only copy the first 440
|
|
# bytes, as these contain the bootstrapping code. The rest of the
|
|
# first sector contains the partition table, and must not be
|
|
# overwritten.
|
|
image_f.write(boot_f.read(440))
|
|
|
|
with open(core_path, "rb") as core_f:
|
|
if platform == "powerpc-ieee1275":
|
|
# write the core to the PrEP partition
|
|
grub2_write_core_prep_part(core_f, image_f, pt)
|
|
else:
|
|
# embed the core in the MBR gap
|
|
grub2_write_core_mbrgap(core_f, image_f, pt)
|
|
|
|
|
|
def main(tree, output_dir, options, loop_client):
|
|
fmt = options["format"]
|
|
filename = options["filename"]
|
|
size = options["size"]
|
|
bootloader = options.get("bootloader", {"type": "grub2"})
|
|
|
|
# sfdisk works on sectors of 512 bytes and ignores excess space - be explicit about this
|
|
if size % 512 != 0:
|
|
raise ValueError("`size` must be a multiple of sector size (512)")
|
|
|
|
if fmt not in ["raw", "raw.xz", "qcow2", "vdi", "vmdk", "vpc"]:
|
|
raise ValueError("`format` must be one of raw, qcow, vdi, vmdk, vpc")
|
|
|
|
image = "/var/tmp/osbuild-image.raw"
|
|
|
|
# Create an empty image file
|
|
subprocess.run(["truncate", "--size", str(size), image], check=True)
|
|
|
|
# The partition table
|
|
pt = partition_table_from_options(options)
|
|
pt.write_to(image)
|
|
|
|
# Install the bootloader
|
|
if bootloader["type"] == "grub2":
|
|
install_grub2(image, pt, bootloader)
|
|
|
|
# Now assemble the filesystem hierarchy and copy the tree into the image
|
|
with contextlib.ExitStack() as cm:
|
|
root = cm.enter_context(tempfile.TemporaryDirectory(prefix="osbuild-mnt"))
|
|
# iterate the partition according to their position in the filesystem tree
|
|
for partition in pt.partitions_with_filesystems():
|
|
offset, size = partition.start_in_bytes, partition.size_in_bytes
|
|
loop = cm.enter_context(loop_client.device(image, offset, size))
|
|
# make the specified filesystem, if any
|
|
if partition.filesystem is None:
|
|
continue
|
|
partition.filesystem.make_at(loop)
|
|
# now mount it
|
|
mountpoint = os.path.normpath(f"{root}/{partition.mountpoint}")
|
|
os.makedirs(mountpoint, exist_ok=True)
|
|
cm.enter_context(mount(loop, mountpoint))
|
|
# the filesystem tree should now be properly setup,
|
|
# copy the tree into the target image
|
|
subprocess.run(["cp", "-a", f"{tree}/.", root], check=True)
|
|
|
|
if fmt == "raw":
|
|
subprocess.run(["cp", image, f"{output_dir}/{filename}"], check=True)
|
|
elif fmt == "raw.xz":
|
|
with open(f"{output_dir}/{filename}", "w") as f:
|
|
subprocess.run(["xz", "--keep", "--stdout", "-0", image], stdout=f, check=True)
|
|
else:
|
|
extra_args = {
|
|
"qcow2": ["-c"],
|
|
"vdi": [],
|
|
"vmdk": ["-c"],
|
|
"vpc": ["-o", "subformat=fixed,force_size"]
|
|
}
|
|
subprocess.run([
|
|
"qemu-img",
|
|
"convert",
|
|
"-O", fmt,
|
|
*extra_args[fmt],
|
|
image,
|
|
f"{output_dir}/{filename}"
|
|
], check=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
args = json.load(sys.stdin)
|
|
|
|
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
|
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
|
|
sock.connect("/run/osbuild/api/remoteloop")
|
|
ret = main(args["tree"], args["output_dir"], args["options"], remoteloop.LoopClient(sock))
|
|
|
|
sys.exit(ret)
|