5ee68aef30
Instead of having dictionaries representing the partition table, partitions and filesystems together with some functions operating on them, have proper python objects with methods. In the future these objects could be extract and properly tested as well.
442 lines
14 KiB
Python
Executable file
442 lines
14 KiB
Python
Executable file
#!/usr/bin/python3
|
|
|
|
import contextlib
|
|
import json
|
|
import os
|
|
import socket
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from typing import List
|
|
import osbuild.remoteloop as remoteloop
|
|
|
|
STAGE_DESC = "Assemble a bootable partitioned disk image with qemu-img"
|
|
STAGE_INFO = """
|
|
Assemble a bootable partitioned disk image using `qemu-img`.
|
|
|
|
Creates a sparse partitioned disk image of type `pttype` of a given `size`,
|
|
with a partition table according to `partitions` or a MBR partitioned disk
|
|
having a single bootable partition containing the root filesystem if the
|
|
`pttype` property is absent.
|
|
|
|
If the partition type is MBR it installs GRUB2 (using the buildhost's
|
|
`/usr/lib/grub/i386-pc/boot.img` etc.) as the bootloader.
|
|
|
|
Copies the tree contents into the root filesystem and then converts the raw
|
|
sparse image into the format requested with the `fmt` option.
|
|
|
|
Buildhost commands used: `truncate`, `mount`, `umount`, `sfdisk`,
|
|
`grub2-mkimage`, `mkfs.ext4` or `mkfs.xfs`, `qemu-img`.
|
|
"""
|
|
STAGE_OPTS = """
|
|
"required": ["format", "filename", "ptuuid", "size"],
|
|
"oneOf": [{
|
|
"required": ["root_fs_uuid"]
|
|
},{
|
|
"required": ["pttype", "partitions"]
|
|
}],
|
|
"properties": {
|
|
"format": {
|
|
"description": "Image file format to use",
|
|
"type": "string",
|
|
"enum": ["raw", "qcow2", "vdi", "vmdk", "vpc"]
|
|
},
|
|
"filename": {
|
|
"description": "Image filename",
|
|
"type": "string"
|
|
},
|
|
"partitions": {
|
|
"description": "Partition layout ",
|
|
"type": "array",
|
|
"items": {
|
|
"description": "Description of one partition",
|
|
"type": "object",
|
|
"properties": {
|
|
"bootable": {
|
|
"description": "Mark the partition as bootable (MBR)",
|
|
"type": "boolean"
|
|
},
|
|
"size": {
|
|
"description": "The size of this partition",
|
|
"type": "integer"
|
|
},
|
|
"start": {
|
|
"description": "The start offset of this partition",
|
|
"type": "integer"
|
|
},
|
|
"type": {
|
|
"description": "The partition type (UUID or identifier)",
|
|
"type": "string"
|
|
},
|
|
"filesystem": {
|
|
"description": "Description of the filesystem",
|
|
"type": "object",
|
|
"required": ["mountpoint", "type", "uuid"],
|
|
"properties": {
|
|
"label": {
|
|
"description": "Label for the filesystem",
|
|
"type": "string"
|
|
},
|
|
"mountpoint": {
|
|
"description": "Where to mount the partition",
|
|
"type": "string"
|
|
},
|
|
"type": {
|
|
"description": "Type of the filesystem",
|
|
"type": "string",
|
|
"enum": ["ext4", "xfs", "vfat"]
|
|
},
|
|
"uuid": {
|
|
"description": "UUID for the filesystem",
|
|
"type": "string"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"ptuuid": {
|
|
"description": "UUID for the disk image's partition table",
|
|
"type": "string"
|
|
},
|
|
"pttype": {
|
|
"description": "The type of the partition table",
|
|
"type": "string",
|
|
"enum": ["mbr", "gpt"]
|
|
},
|
|
"root_fs_uuid": {
|
|
"description": "UUID for the root filesystem",
|
|
"type": "string"
|
|
},
|
|
"size": {
|
|
"description": "Virtual disk size",
|
|
"type": "string"
|
|
},
|
|
"root_fs_type": {
|
|
"description": "Type of the root filesystem",
|
|
"type": "string",
|
|
"enum": ["ext4", "xfs"],
|
|
"default": "ext4"
|
|
}
|
|
}
|
|
"""
|
|
|
|
@contextlib.contextmanager
|
|
def mount(source, dest):
|
|
subprocess.run(["mount", source, dest], check=True)
|
|
try:
|
|
yield dest
|
|
finally:
|
|
subprocess.run(["umount", "-R", dest], check=True)
|
|
|
|
|
|
class Filesystem:
|
|
def __init__(self,
|
|
fstype: str,
|
|
uuid: str,
|
|
mountpoint: str,
|
|
label: str = None):
|
|
self.type = fstype
|
|
self.uuid = uuid
|
|
self.mountpoint = mountpoint
|
|
self.label = label
|
|
|
|
|
|
def mkfs_ext4(device, uuid, label):
|
|
opts = []
|
|
if label:
|
|
opts = ["-L", label]
|
|
subprocess.run(["mkfs.ext4", "-U", uuid] + opts + [device],
|
|
input="y", encoding='utf-8', check=True)
|
|
|
|
|
|
def mkfs_xfs(device, uuid, label):
|
|
opts = []
|
|
if label:
|
|
opts = ["-L", label]
|
|
subprocess.run(["mkfs.xfs", "-m", f"uuid={uuid}"] + opts + [device],
|
|
encoding='utf-8', check=True)
|
|
|
|
|
|
def mkfs_vfat(device, uuid, label):
|
|
volid = uuid.replace('-', '')
|
|
opts = []
|
|
if label:
|
|
opts = ["-n", label]
|
|
subprocess.run(["mkfs.vfat", "-i", volid] + opts + [device], encoding='utf-8', check=True)
|
|
|
|
|
|
def mkfs_for_type(device: str, fs: Filesystem):
|
|
fs_type = fs.type
|
|
if fs_type == "ext4":
|
|
maker = mkfs_ext4
|
|
elif fs_type == "xfs":
|
|
maker = mkfs_xfs
|
|
elif fs_type == "vfat":
|
|
maker = mkfs_vfat
|
|
else:
|
|
raise ValueError(f"Unknown filesystem type '{fs_type}'")
|
|
maker(device, fs.uuid, fs.label)
|
|
|
|
|
|
class Partition:
|
|
def __init__(self,
|
|
pttype: str = None,
|
|
start: int = None,
|
|
size: int = None,
|
|
bootable: bool = False,
|
|
filesystem: Filesystem = None):
|
|
self.type = pttype
|
|
self.start = start
|
|
self.size = size
|
|
self.bootable = bootable
|
|
self.filesystem = filesystem
|
|
|
|
@property
|
|
def start_in_bytes(self):
|
|
return (self.start or 0) * 512
|
|
|
|
@property
|
|
def size_in_bytes(self):
|
|
return (self.size or 0) * 512
|
|
|
|
@property
|
|
def mountpoint(self):
|
|
if self.filesystem is None:
|
|
return None
|
|
return self.filesystem.mountpoint
|
|
|
|
@property
|
|
def fs_type(self):
|
|
if self.filesystem is None:
|
|
return None
|
|
return self.filesystem.type
|
|
|
|
@property
|
|
def fs_uuid(self):
|
|
if self.filesystem is None:
|
|
return None
|
|
return self.filesystem.uuid
|
|
|
|
|
|
class PartitionTable:
|
|
def __init__(self, label, uuid, partitions):
|
|
self.label = label
|
|
self.uuid = uuid
|
|
self.partitions = partitions or []
|
|
|
|
def __getitem__(self, key) -> Partition:
|
|
return self.partitions[key]
|
|
|
|
def partitions_with_filesystems(self) -> List[Partition]:
|
|
"""Return partitions with filesystems sorted by hierarchy"""
|
|
def mountpoint_len(p):
|
|
return len(p.mountpoint)
|
|
parts_fs = filter(lambda p: p.filesystem is not None, self.partitions)
|
|
return sorted(parts_fs, key=mountpoint_len)
|
|
|
|
def write_to(self, target, sync=True):
|
|
"""Write the partition table to disk"""
|
|
# generate the command for sfdisk to create the table
|
|
command = f"label: {self.label}\nlabel-id: {self.uuid}"
|
|
for partition in self.partitions:
|
|
fields = []
|
|
for field in ["start", "size", "type"]:
|
|
value = getattr(partition, field)
|
|
if value:
|
|
fields += [f'{field}="{value}"']
|
|
if partition.bootable:
|
|
fields += ["bootable"]
|
|
command += "\n" + ", ".join(fields)
|
|
|
|
subprocess.run(["sfdisk", "-q", target],
|
|
input=command,
|
|
encoding='utf-8',
|
|
check=True)
|
|
|
|
if sync:
|
|
self.update_from(target)
|
|
|
|
def update_from(self, target):
|
|
"""Update and fill in missing information from disk"""
|
|
r = subprocess.run(["sfdisk", "--json", target],
|
|
stdout=subprocess.PIPE,
|
|
encoding='utf-8',
|
|
check=True)
|
|
disk_table = json.loads(r.stdout)["partitiontable"]
|
|
disk_parts = disk_table["partitions"]
|
|
|
|
assert len(disk_parts) == len(self.partitions)
|
|
for i, part in enumerate(self.partitions):
|
|
part.start = disk_parts[i]["start"]
|
|
part.size = disk_parts[i]["size"]
|
|
part.type = disk_parts[i].get("type")
|
|
|
|
|
|
def filesystem_from_json(js) -> Filesystem:
|
|
return Filesystem(js["type"], js["uuid"], js["mountpoint"], js.get("label"))
|
|
|
|
|
|
def partition_from_json(js) -> Partition:
|
|
p = Partition(pttype=js.get("type"),
|
|
start=js.get("start"),
|
|
size=js.get("size"),
|
|
bootable=js.get("bootable"))
|
|
fs = js.get("filesystem")
|
|
if fs:
|
|
p.filesystem = filesystem_from_json(fs)
|
|
return p
|
|
|
|
|
|
def partition_table_from_options(options) -> PartitionTable:
|
|
ptuuid = options["ptuuid"]
|
|
pttype = options.get("pttype", "dos")
|
|
partitions = options.get("partitions")
|
|
|
|
if pttype == "mbr":
|
|
pttype = "dos"
|
|
|
|
if partitions is None:
|
|
# legacy mode, create a correct
|
|
root_fs_uuid = options["root_fs_uuid"]
|
|
root_fs_type = options.get("root_fs_type", "ext4")
|
|
partitions = [{
|
|
"bootable": True,
|
|
"type": "83",
|
|
"filesystem": {
|
|
"type": root_fs_type,
|
|
"uuid": root_fs_uuid,
|
|
"mountpoint": "/"
|
|
}
|
|
}]
|
|
parts = [partition_from_json(p) for p in partitions]
|
|
return PartitionTable(pttype, ptuuid, parts)
|
|
|
|
|
|
def install_grub2(image: str, pt: PartitionTable):
|
|
"""Install grub2 to image"""
|
|
grub2_core = "/var/tmp/grub2-core.img"
|
|
|
|
root_fs_type = "unknown"
|
|
for p in pt:
|
|
if p.mountpoint and p.mountpoint == "/":
|
|
root_fs_type = p.fs_type
|
|
break
|
|
|
|
if root_fs_type == "ext4":
|
|
fs_module = "ext2"
|
|
elif root_fs_type == "xfs":
|
|
fs_module = "xfs"
|
|
else:
|
|
raise ValueError(f"unknown root filesystem type: '{root_fs_type}'")
|
|
|
|
# Create the level-2 bootloader
|
|
# The purpose of this is to find the grub modules and configuration
|
|
# to be able to start the level-3 bootloader. It contains the modules
|
|
# necessary to do this, but nothing else.
|
|
subprocess.run(["grub2-mkimage",
|
|
"--verbose",
|
|
"--directory", "/usr/lib/grub/i386-pc",
|
|
"--prefix", "(,msdos1)/boot/grub2",
|
|
"--format", "i386-pc",
|
|
"--compression", "auto",
|
|
"--output", grub2_core,
|
|
"part_msdos", fs_module, "biosdisk"],
|
|
check=True)
|
|
|
|
partition_offset = pt[0].start_in_bytes
|
|
assert os.path.getsize(grub2_core) < partition_offset - 512
|
|
|
|
with open(image, "rb+") as image_f:
|
|
# Install the level-1 bootloader into the start of the MBR
|
|
# The purpose of this is simply to jump into the level-2 bootloader.
|
|
with open("/usr/lib/grub/i386-pc/boot.img", "rb") as boot_f:
|
|
# The boot.img file is 512 bytes, but we must only copy the first 440
|
|
# bytes, as these contain the bootstrapping code. The rest of the
|
|
# first sector contains the partition table, and must not be
|
|
# overwritten.
|
|
image_f.write(boot_f.read(440))
|
|
|
|
# Install the level-2 bootloader into the space after the MBR, before
|
|
# the first partition.
|
|
with open(grub2_core, "rb") as core_f:
|
|
image_f.seek(512)
|
|
shutil.copyfileobj(core_f, image_f)
|
|
|
|
|
|
def main(tree, output_dir, options, loop_client):
|
|
fmt = options["format"]
|
|
filename = options["filename"]
|
|
size = options["size"]
|
|
|
|
# sfdisk works on sectors of 512 bytes and ignores excess space - be explicit about this
|
|
if size % 512 != 0:
|
|
raise ValueError("`size` must be a multiple of sector size (512)")
|
|
|
|
if fmt not in ["raw", "raw.xz", "qcow2", "vdi", "vmdk", "vpc"]:
|
|
raise ValueError("`format` must be one of raw, qcow, vdi, vmdk, vpc")
|
|
|
|
image = "/var/tmp/osbuild-image.raw"
|
|
|
|
# Create an empty image file
|
|
subprocess.run(["truncate", "--size", str(size), image], check=True)
|
|
|
|
# The partition table
|
|
pt = partition_table_from_options(options)
|
|
pt.write_to(image)
|
|
|
|
# Create the level-2 bootloader
|
|
if pt.label == "dos":
|
|
install_grub2(image, pt)
|
|
|
|
# Now assemble the filesystem hierarchy and copy the tree into the image
|
|
with contextlib.ExitStack() as cm:
|
|
root = cm.enter_context(tempfile.TemporaryDirectory(prefix="osbuild-mnt"))
|
|
# iterate the partition according to their position in the filesystem tree
|
|
for partition in pt.partitions_with_filesystems():
|
|
offset, size = partition.start_in_bytes, partition.size_in_bytes
|
|
loop = cm.enter_context(loop_client.device(image, offset, size))
|
|
# make the specified filesystem, if any
|
|
if partition.filesystem is None:
|
|
continue
|
|
mkfs_for_type(loop, partition.filesystem)
|
|
# now mount it
|
|
mountpoint = os.path.normpath(f"{root}/{partition.mountpoint}")
|
|
os.makedirs(mountpoint, exist_ok=True)
|
|
cm.enter_context(mount(loop, mountpoint))
|
|
# the filesystem tree should now be properly setup,
|
|
# copy the tree into the target image
|
|
subprocess.run(["cp", "-a", f"{tree}/.", root], check=True)
|
|
|
|
if fmt == "raw":
|
|
subprocess.run(["cp", image, f"{output_dir}/{filename}"], check=True)
|
|
elif fmt == "raw.xz":
|
|
with open(f"{output_dir}/{filename}", "w") as f:
|
|
subprocess.run(["xz", "--keep", "--stdout", "-0", image], stdout=f, check=True)
|
|
else:
|
|
extra_args = {
|
|
"qcow2": ["-c"],
|
|
"vdi": [],
|
|
"vmdk": ["-c"],
|
|
"vpc": ["-o", "subformat=fixed,force_size"]
|
|
}
|
|
subprocess.run([
|
|
"qemu-img",
|
|
"convert",
|
|
"-O", fmt,
|
|
*extra_args[fmt],
|
|
image,
|
|
f"{output_dir}/{filename}"
|
|
], check=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
args = json.load(sys.stdin)
|
|
|
|
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
|
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
|
|
sock.connect("/run/osbuild/api/remoteloop")
|
|
ret = main(args["tree"], args["output_dir"], args["options"], remoteloop.LoopClient(sock))
|
|
|
|
sys.exit(ret)
|