No description
bf41326ac6
Using O_DIRECT to open the image partition and then using that fd
for the backing of the loopback device will break the mounting of
the formatted partition, i.e mount will fail with:
mount: /tmp/looptest-6qrtkp5e/mountpoint-root: wrong fs type,
bad option, bad superblock on /dev/loop0, missing codepage or
helper program, or other error.
Reproducible with the follow small-ish python script, executed via
'env PYTHONPATH=$(pwd) python3 looptest.py':
---- 8< ---- 8< ---- [ looptest.py ] ---- 8< ---- 8< ----
import contextlib
import json
import os
import subprocess
import stat
import tempfile
from osbuild import loop
@contextlib.contextmanager
def mount(source, dest):
subprocess.run(["mount", source, dest], check=True)
try:
yield dest
finally:
subprocess.run(["umount", "-R", dest], check=True)
@contextlib.contextmanager
def os_open(path, flags):
fd = os.open(path, flags)
try:
yield fd
finally:
os.close(fd)
def main():
size = 512 * 1024 * 1024
ptuuid = "0x14fc63d2"
with contextlib.ExitStack() as cm:
tmpdir = cm.enter_context(tempfile.TemporaryDirectory(prefix="looptest-"))
print(f"Temporary directory at {tmpdir}")
devdir = os.path.join(tmpdir, "dev")
os.makedirs(devdir, exist_ok=True)
dir_fd = cm.enter_context(os_open(devdir, os.O_DIRECTORY))
image = os.path.join(tmpdir, "image")
subprocess.run(["truncate", "--size", str(size), image], check=True)
table = f"label: mbr\nlabel-id: {ptuuid}\nbootable, type=83"
subprocess.run(["sfdisk", image], input=table, encoding='utf-8',
check=True)
# read it back
r = subprocess.run(["sfdisk", "--json", image],
stdout=subprocess.PIPE,
encoding='utf-8', check=True)
table = json.loads(r.stdout)["partitiontable"]
partitions = table["partitions"]
start = partitions[0]["start"] * 512
size = partitions[0]["size"] * 512
# fails here with os.O_DIRECT
image_fd = cm.enter_context(os_open(image, os.O_RDWR | os.O_DIRECT))
control = loop.LoopControl()
minor = control.get_unbound()
lo = loop.Loop(minor)
lo.set_fd(image_fd)
lo.set_status(offset=start, sizelimit=size, autoclear=True)
lo.mknod(dir_fd)
loopdev = f"/dev/loop{minor}"
# loopdev = os.path.join(devdir, lo.devname)
# os.chmod(loopdev, os.stat(loopdev).st_mode | stat.S_IRGRP)
subprocess.run(["ls", "-la", f"{devdir}"], check=True)
subprocess.run(["mkfs.ext4", loopdev],
input="y", encoding='utf-8', check=True)
subprocess.run(["blkid", loopdev], check=True)
mountpoint = os.path.join(tmpdir, "mountpoint-root")
os.makedirs(mountpoint, exist_ok=True)
cm.enter_context(mount(loopdev, mountpoint))
subprocess.run(["ls", "-la", tmpdir], check=True)
subprocess.run(["ls", "-la", mountpoint], check=True)
subprocess.run(["mount"], check=True)
if __name__ == '__main__':
main()
|
||
|---|---|---|
| assemblers | ||
| osbuild | ||
| runners | ||
| samples | ||
| sources | ||
| stages | ||
| test | ||
| .editorconfig | ||
| .gitignore | ||
| .packit.yaml | ||
| .pylintrc | ||
| .travis.yml | ||
| bump-version.sh | ||
| LICENSE | ||
| Makefile | ||
| MANIFEST.in | ||
| osbuild.spec | ||
| README.md | ||
| RELEASE.md | ||
| setup.py | ||
| tree-diff | ||
osbuild
A build system for operating system images, working towards an image build pipeline that's more comprehensible, reproducible, and extendable.
Pipelines
The build process for an image is described by a pipeline. Each stage in a pipeline is a program that, given some configuration, modifies a file system tree. Finally, an assembler takes a filesystem tree, and assembles it into an image. Pipelines are defined as JSON files like this one:
{
"name": "Example Image",
"stages": [
{
"name": "org.osbuild.dnf",
"options": {
"releasever": "30",
"basearch": "x86_64",
"repos": [
{
"metalink": "https://mirrors.fedoraproject.org/metalink?repo=fedora-$releasever&arch=$basearch",
"checksum": "sha256:9f596e18f585bee30ac41c11fb11a83ed6b11d5b341c1cb56ca4015d7717cb97",
"gpgkey": "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nmQINBFturGcBEACv0xBo91V2n0uEC2vh69ywCiSyvUgN/AQH8EZpCVtM7NyjKgKm\nbbY4G3R0M3ir1xXmvUDvK0493/qOiFrjkplvzXFTGpPTi0ypqGgxc5d0ohRA1M75\nL+0AIlXoOgHQ358/c4uO8X0JAA1NYxCkAW1KSJgFJ3RjukrfqSHWthS1d4o8fhHy\nKJKEnirE5hHqB50dafXrBfgZdaOs3C6ppRIePFe2o4vUEapMTCHFw0woQR8Ah4/R\nn7Z9G9Ln+0Cinmy0nbIDiZJ+pgLAXCOWBfDUzcOjDGKvcpoZharA07c0q1/5ojzO\n4F0Fh4g/BUmtrASwHfcIbjHyCSr1j/3Iz883iy07gJY5Yhiuaqmp0o0f9fgHkG53\n2xCU1owmACqaIBNQMukvXRDtB2GJMuKa/asTZDP6R5re+iXs7+s9ohcRRAKGyAyc\nYKIQKcaA+6M8T7/G+TPHZX6HJWqJJiYB+EC2ERblpvq9TPlLguEWcmvjbVc31nyq\nSDoO3ncFWKFmVsbQPTbP+pKUmlLfJwtb5XqxNR5GEXSwVv4I7IqBmJz1MmRafnBZ\ng0FJUtH668GnldO20XbnSVBr820F5SISMXVwCXDXEvGwwiB8Lt8PvqzXnGIFDAu3\nDlQI5sxSqpPVWSyw08ppKT2Tpmy8adiBotLfaCFl2VTHwOae48X2dMPBvQARAQAB\ntDFGZWRvcmEgKDMwKSA8ZmVkb3JhLTMwLXByaW1hcnlAZmVkb3JhcHJvamVjdC5v\ncmc+iQI4BBMBAgAiBQJbbqxnAhsPBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAK\nCRDvPBEfz8ZZudTnD/9170LL3nyTVUCFmBjT9wZ4gYnpwtKVPa/pKnxbbS+Bmmac\ng9TrT9pZbqOHrNJLiZ3Zx1Hp+8uxr3Lo6kbYwImLhkOEDrf4aP17HfQ6VYFbQZI8\nf79OFxWJ7si9+3gfzeh9UYFEqOQfzIjLWFyfnas0OnV/P+RMQ1Zr+vPRqO7AR2va\nN9wg+Xl7157dhXPCGYnGMNSoxCbpRs0JNlzvJMuAea5nTTznRaJZtK/xKsqLn51D\nK07k9MHVFXakOH8QtMCUglbwfTfIpO5YRq5imxlWbqsYWVQy1WGJFyW6hWC0+RcJ\nOx5zGtOfi4/dN+xJ+ibnbyvy/il7Qm+vyFhCYqIPyS5m2UVJUuao3eApE38k78/o\n8aQOTnFQZ+U1Sw+6woFTxjqRQBXlQm2+7Bt3bqGATg4sXXWPbmwdL87Ic+mxn/ml\nSMfQux/5k6iAu1kQhwkO2YJn9eII6HIPkW+2m5N1JsUyJQe4cbtZE5Yh3TRA0dm7\n+zoBRfCXkOW4krchbgww/ptVmzMMP7GINJdROrJnsGl5FVeid9qHzV7aZycWSma7\nCxBYB1J8HCbty5NjtD6XMYRrMLxXugvX6Q4NPPH+2NKjzX4SIDejS6JjgrP3KA3O\npMuo7ZHMfveBngv8yP+ZD/1sS6l+dfExvdaJdOdgFCnp4p3gPbw5+Lv70HrMjA==\n=BfZ/\n-----END PGP PUBLIC KEY BLOCK-----\n"
}
],
"packages": [ "@Core", "grub2-pc", "httpd" ]
}
},
{
"name": "org.osbuild.systemd",
"options": {
"enabled_services": [ "httpd" ]
}
},
{
"name": "org.osbuild.grub2",
"options": {
"root_fs_uuid": "76a22bf4-f153-4541-b6c7-0332c0dfaeac"
}
}
],
"assembler": {
"name": "org.osbuild.qemu",
"options": {
"format": "qcow2",
"filename": "example.qcow2",
"ptuuid": "0x7e83a7ba",
"root_fs_uuid": "76a22bf4-f153-4541-b6c7-0332c0dfaeac",
"size": 3221225472
}
}
}
osbuild runs each of the stages in turn, isolating them from the host and
from each other, with the exception that they all operate on the same
filesystem-tree. The assembler is similarly isolated, and given the same
tree, in read-only mode and assembles it into an image without altering
its contents.
The filesystem tree produced by the final stage of a pipeline, is named and optionally saved to be reused as the base for future pipelines.
Each stage is passed the (appended) options object as JSON over stdin.
The above pipeline has no base and produces a qcow2 image.
Running
usage: __main__.py [-h] [--build-env ENV] [--store DIRECTORY] [-l DIRECTORY]
[--json]
PIPELINE
Build operating system images
positional arguments:
PIPELINE json file containing the pipeline that should be
built, or a '-' to read from stdin
optional arguments:
-h, --help show this help message and exit
--build-env ENV json file containing a description of the build
environment
--store DIRECTORY the directory where intermediary os trees are stored
--sources SOURCES json file containing a dictionary of source
configuration
-l DIRECTORY, --libdir DIRECTORY
the directory containing stages, assemblers, and the
osbuild library
--json output results in JSON format
Running example
You can build basic qcow2 image of Fedora 30 by running a following command:
sudo python3 -m osbuild --libdir . --sources samples/sources.json samples/base-qcow2.json
-
Root rights are required because osbuild heavily relies on creating systemd containers and bind mounting.
It shouldn't interfere with host OS but please be careful! It's still under development!
-
--libdirargument is required becauseosbuildexpects itself to be installed in directories under/usr. Using this argument you can change the expected path. -
You don't need to use any kind of virtual environment, modern version of Python 3 is enough.
osbuilduses only standard library and linux commands.