Introduce runners
We've been using a generic `osbuild-run`, which sets up the build environment (and works around bugs) for all build roots. It is already getting unwieldy, because it tries to detect the OS for some things it configures. It's also about to cause problems for RHEL, which doesn't currently support a python3 shebang without having /etc around. This patch changes the `build` key in a pipeline to not be a pipeline itself, but an object with `runner` and `pipeline` keys. `pipeline` is the build pipeline, as before. `runner` is the name of the runner to use. Runners are programs in the `runners` subdirectory. Three runners are included in this patch. They're copies of osbuild-run for now (except some additions for rhel82). The idea is that each of them only contains the minimal setup code necessary for an OS, and that we can review what's needed when updating a build root. Also modify the `--build-pipeline` command line switch to accept such a build object (instead of a pipeline) and rename it accordingly, to `--build-env`. Correspondingly, `OSBUILD_TEST_BUILD_PIPELINE` → `OSBUILD_TEST_BUILD_ENV`.
This commit is contained in:
Lars Karlitski
Tom Gundersen
parent
616e1ecbba
commit
64713449ce
38 changed files with 969 additions and 498 deletions
112
runners/org.osbuild.fedora27
Executable file
112
runners/org.osbuild.fedora27
Executable file
|
|
@ -0,0 +1,112 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import array
|
||||
import json
|
||||
import shutil
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
# copied from remoteloop.py
|
||||
def load_fds(sock, msglen):
|
||||
fds = array.array("i") # Array of ints
|
||||
msg, ancdata, _, addr = sock.recvmsg(msglen, socket.CMSG_LEN(253 * fds.itemsize))
|
||||
for cmsg_level, cmsg_type, cmsg_data in ancdata:
|
||||
if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS):
|
||||
# Append data, ignoring any truncated integers at the end.
|
||||
fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
|
||||
return json.loads(msg), list(fds), addr
|
||||
|
||||
def ldconfig():
|
||||
# ld.so.conf must exist, or `ldconfig` throws a warning
|
||||
subprocess.run(["touch", "/etc/ld.so.conf"], check=True)
|
||||
subprocess.run(["ldconfig"], check=True)
|
||||
|
||||
|
||||
def sysusers():
|
||||
try:
|
||||
subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
|
||||
except subprocess.CalledProcessError as error:
|
||||
sys.stderr.write(error.stdout)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def update_ca_trust():
|
||||
if not shutil.which("update-ca-trust"):
|
||||
return
|
||||
|
||||
# generate /etc/pki/tls/certs/ca-bundle.crt
|
||||
os.makedirs("/etc/pki/ca-trust/extracted/pem")
|
||||
os.makedirs("/etc/pki/tls/certs")
|
||||
os.symlink("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "/etc/pki/tls/certs/ca-bundle.crt")
|
||||
|
||||
# allow to fail, because it sometimes mysteriously does
|
||||
subprocess.run(["update-ca-trust", "extract"], check=False)
|
||||
|
||||
|
||||
def append_certs(cert_conf, dir_fd, parents=b""):
|
||||
for entry in os.scandir(f"/proc/self/fd/{dir_fd}".encode()):
|
||||
if entry.is_file():
|
||||
line = os.path.join(parents, entry.name)
|
||||
cert_conf.write(line)
|
||||
cert_conf.write(b"\n")
|
||||
elif entry.is_dir():
|
||||
append_certs(cert_conf,
|
||||
os.open(entry.name, os.O_DIRECTORY, dir_fd=dir_fd),
|
||||
os.path.join(parents, entry.name))
|
||||
|
||||
|
||||
def update_ca_certificates():
|
||||
if not shutil.which("update-ca-certificates"):
|
||||
return
|
||||
|
||||
# generate /etc/ssl/certs/ca-certificates.crt
|
||||
os.makedirs("/etc/ssl/certs")
|
||||
with open("/etc/ca-certificates.conf", "wb") as f:
|
||||
append_certs(f, os.open("/usr/share/ca-certificates", os.O_DIRECTORY))
|
||||
subprocess.run(["update-ca-certificates"], check=True)
|
||||
|
||||
|
||||
def tmpfiles():
|
||||
# Allow systemd-tmpfiles to return non-0. Some packages want to create
|
||||
# directories owned by users that are not set up with systemd-sysusers.
|
||||
subprocess.run(["systemd-tmpfiles", "--create"], check=False)
|
||||
|
||||
|
||||
def nsswitch():
|
||||
# the default behavior is fine, but using nss-resolve does not
|
||||
# necessarily work in a non-booted container, so make sure that
|
||||
# is not configured.
|
||||
try:
|
||||
os.remove("/etc/nsswitch.conf")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def setup_stdio():
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
|
||||
sock.connect("/run/osbuild/api/osbuild")
|
||||
req = {'method': 'setup-stdio'}
|
||||
sock.send(json.dumps(req).encode('utf-8'))
|
||||
msg, fds, _ = load_fds(sock, 1024)
|
||||
for io in ['stdin', 'stdout', 'stderr']:
|
||||
target = getattr(sys, io)
|
||||
source = fds[msg[io]]
|
||||
os.dup2(source, target.fileno())
|
||||
os.close(source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
setup_stdio()
|
||||
ldconfig()
|
||||
sysusers()
|
||||
update_ca_trust()
|
||||
update_ca_certificates()
|
||||
tmpfiles()
|
||||
nsswitch()
|
||||
|
||||
r = subprocess.run(sys.argv[1:], check=False)
|
||||
sys.exit(r.returncode)
|
||||
112
runners/org.osbuild.fedora30
Executable file
112
runners/org.osbuild.fedora30
Executable file
|
|
@ -0,0 +1,112 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import array
|
||||
import json
|
||||
import shutil
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
# copied from remoteloop.py
|
||||
def load_fds(sock, msglen):
|
||||
fds = array.array("i") # Array of ints
|
||||
msg, ancdata, _, addr = sock.recvmsg(msglen, socket.CMSG_LEN(253 * fds.itemsize))
|
||||
for cmsg_level, cmsg_type, cmsg_data in ancdata:
|
||||
if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS):
|
||||
# Append data, ignoring any truncated integers at the end.
|
||||
fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
|
||||
return json.loads(msg), list(fds), addr
|
||||
|
||||
def ldconfig():
|
||||
# ld.so.conf must exist, or `ldconfig` throws a warning
|
||||
subprocess.run(["touch", "/etc/ld.so.conf"], check=True)
|
||||
subprocess.run(["ldconfig"], check=True)
|
||||
|
||||
|
||||
def sysusers():
|
||||
try:
|
||||
subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
|
||||
except subprocess.CalledProcessError as error:
|
||||
sys.stderr.write(error.stdout)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def update_ca_trust():
|
||||
if not shutil.which("update-ca-trust"):
|
||||
return
|
||||
|
||||
# generate /etc/pki/tls/certs/ca-bundle.crt
|
||||
os.makedirs("/etc/pki/ca-trust/extracted/pem")
|
||||
os.makedirs("/etc/pki/tls/certs")
|
||||
os.symlink("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "/etc/pki/tls/certs/ca-bundle.crt")
|
||||
|
||||
# allow to fail, because it sometimes mysteriously does
|
||||
subprocess.run(["update-ca-trust", "extract"], check=False)
|
||||
|
||||
|
||||
def append_certs(cert_conf, dir_fd, parents=b""):
|
||||
for entry in os.scandir(f"/proc/self/fd/{dir_fd}".encode()):
|
||||
if entry.is_file():
|
||||
line = os.path.join(parents, entry.name)
|
||||
cert_conf.write(line)
|
||||
cert_conf.write(b"\n")
|
||||
elif entry.is_dir():
|
||||
append_certs(cert_conf,
|
||||
os.open(entry.name, os.O_DIRECTORY, dir_fd=dir_fd),
|
||||
os.path.join(parents, entry.name))
|
||||
|
||||
|
||||
def update_ca_certificates():
|
||||
if not shutil.which("update-ca-certificates"):
|
||||
return
|
||||
|
||||
# generate /etc/ssl/certs/ca-certificates.crt
|
||||
os.makedirs("/etc/ssl/certs")
|
||||
with open("/etc/ca-certificates.conf", "wb") as f:
|
||||
append_certs(f, os.open("/usr/share/ca-certificates", os.O_DIRECTORY))
|
||||
subprocess.run(["update-ca-certificates"], check=True)
|
||||
|
||||
|
||||
def tmpfiles():
|
||||
# Allow systemd-tmpfiles to return non-0. Some packages want to create
|
||||
# directories owned by users that are not set up with systemd-sysusers.
|
||||
subprocess.run(["systemd-tmpfiles", "--create"], check=False)
|
||||
|
||||
|
||||
def nsswitch():
|
||||
# the default behavior is fine, but using nss-resolve does not
|
||||
# necessarily work in a non-booted container, so make sure that
|
||||
# is not configured.
|
||||
try:
|
||||
os.remove("/etc/nsswitch.conf")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def setup_stdio():
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
|
||||
sock.connect("/run/osbuild/api/osbuild")
|
||||
req = {'method': 'setup-stdio'}
|
||||
sock.send(json.dumps(req).encode('utf-8'))
|
||||
msg, fds, _ = load_fds(sock, 1024)
|
||||
for io in ['stdin', 'stdout', 'stderr']:
|
||||
target = getattr(sys, io)
|
||||
source = fds[msg[io]]
|
||||
os.dup2(source, target.fileno())
|
||||
os.close(source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
setup_stdio()
|
||||
ldconfig()
|
||||
sysusers()
|
||||
update_ca_trust()
|
||||
update_ca_certificates()
|
||||
tmpfiles()
|
||||
nsswitch()
|
||||
|
||||
r = subprocess.run(sys.argv[1:], check=False)
|
||||
sys.exit(r.returncode)
|
||||
147
runners/org.osbuild.rhel82
Executable file
147
runners/org.osbuild.rhel82
Executable file
|
|
@ -0,0 +1,147 @@
|
|||
#!/usr/bin/python3.6
|
||||
|
||||
import array
|
||||
import json
|
||||
import shutil
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
# copied from remoteloop.py
|
||||
def load_fds(sock, msglen):
|
||||
fds = array.array("i") # Array of ints
|
||||
msg, ancdata, _, addr = sock.recvmsg(msglen, socket.CMSG_LEN(253 * fds.itemsize))
|
||||
for cmsg_level, cmsg_type, cmsg_data in ancdata:
|
||||
if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS):
|
||||
# Append data, ignoring any truncated integers at the end.
|
||||
fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
|
||||
return json.loads(msg), list(fds), addr
|
||||
|
||||
def ldconfig():
|
||||
# ld.so.conf must exist, or `ldconfig` throws a warning
|
||||
subprocess.run(["touch", "/etc/ld.so.conf"], check=True)
|
||||
subprocess.run(["ldconfig"], check=True)
|
||||
|
||||
|
||||
def sysusers():
|
||||
try:
|
||||
subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
|
||||
except subprocess.CalledProcessError as error:
|
||||
sys.stderr.write(error.stdout)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def update_ca_trust():
|
||||
if not shutil.which("update-ca-trust"):
|
||||
return
|
||||
|
||||
# generate /etc/pki/tls/certs/ca-bundle.crt
|
||||
os.makedirs("/etc/pki/ca-trust/extracted/pem")
|
||||
os.makedirs("/etc/pki/tls/certs")
|
||||
os.symlink("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "/etc/pki/tls/certs/ca-bundle.crt")
|
||||
|
||||
# allow to fail, because it sometimes mysteriously does
|
||||
subprocess.run(["update-ca-trust", "extract"], check=False)
|
||||
|
||||
|
||||
def append_certs(cert_conf, dir_fd, parents=b""):
|
||||
for entry in os.scandir(f"/proc/self/fd/{dir_fd}".encode()):
|
||||
if entry.is_file():
|
||||
line = os.path.join(parents, entry.name)
|
||||
cert_conf.write(line)
|
||||
cert_conf.write(b"\n")
|
||||
elif entry.is_dir():
|
||||
append_certs(cert_conf,
|
||||
os.open(entry.name, os.O_DIRECTORY, dir_fd=dir_fd),
|
||||
os.path.join(parents, entry.name))
|
||||
|
||||
|
||||
def update_ca_certificates():
|
||||
if not shutil.which("update-ca-certificates"):
|
||||
return
|
||||
|
||||
# generate /etc/ssl/certs/ca-certificates.crt
|
||||
os.makedirs("/etc/ssl/certs")
|
||||
with open("/etc/ca-certificates.conf", "wb") as f:
|
||||
append_certs(f, os.open("/usr/share/ca-certificates", os.O_DIRECTORY))
|
||||
subprocess.run(["update-ca-certificates"], check=True)
|
||||
|
||||
|
||||
def tmpfiles():
|
||||
# Allow systemd-tmpfiles to return non-0. Some packages want to create
|
||||
# directories owned by users that are not set up with systemd-sysusers.
|
||||
subprocess.run(["systemd-tmpfiles", "--create"], check=False)
|
||||
|
||||
|
||||
def nsswitch():
|
||||
# the default behavior is fine, but using nss-resolve does not
|
||||
# necessarily work in a non-booted container, so make sure that
|
||||
# is not configured.
|
||||
try:
|
||||
os.remove("/etc/nsswitch.conf")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def setup_stdio():
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
|
||||
sock.connect("/run/osbuild/api/osbuild")
|
||||
req = {'method': 'setup-stdio'}
|
||||
sock.send(json.dumps(req).encode('utf-8'))
|
||||
msg, fds, _ = load_fds(sock, 1024)
|
||||
for io in ['stdin', 'stdout', 'stderr']:
|
||||
target = getattr(sys, io)
|
||||
source = fds[msg[io]]
|
||||
os.dup2(source, target.fileno())
|
||||
os.close(source)
|
||||
|
||||
def os_release():
|
||||
"""/usr/lib/os-release doesn't exist. The `redhat-release` package
|
||||
generates `/etc/os-release directly. To work around this, do the same here.
|
||||
|
||||
https://bugzilla.redhat.com/show_bug.cgi?id=1766754
|
||||
"""
|
||||
|
||||
# remove the symlink that systemd-nspawn creates
|
||||
os.remove("/etc/os-release")
|
||||
with open("/etc/os-release", "w") as f:
|
||||
f.write('NAME="Red Hat Enterprise Linux"\n')
|
||||
f.write('VERSION="8.2 (Ootpa)"\n')
|
||||
f.write('ID="rhel"\n')
|
||||
f.write('ID_LIKE="fedora"\n')
|
||||
f.write('VERSION_ID="8.2"\n')
|
||||
f.write('PLATFORM_ID="platform:el8"\n')
|
||||
f.write('PRETTY_NAME="Red Hat Enterprise Linux 8.2 Beta (Ootpa)"\n')
|
||||
f.write('ANSI_COLOR="0;31"\n')
|
||||
f.write('CPE_NAME="cpe:/o:redhat:enterprise_linux:8.2:beta"\n')
|
||||
f.write('HOME_URL="https://www.redhat.com/"\n')
|
||||
f.write('BUG_REPORT_URL="https://bugzilla.redhat.com/"\n')
|
||||
|
||||
|
||||
def python_alternatives():
|
||||
"""/usr/bin/python3 is a symlink to /etc/alternatives/python3, which points
|
||||
to /usr/bin/python3.6 by default. Recreate the link in /etc, so that
|
||||
shebang lines in stages and assemblers work.
|
||||
"""
|
||||
os.makedirs("/etc/alternatives", exist_ok=True)
|
||||
try:
|
||||
os.symlink("/usr/bin/python3.6", "/etc/alternatives/python3")
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
setup_stdio()
|
||||
ldconfig()
|
||||
sysusers()
|
||||
update_ca_trust()
|
||||
update_ca_certificates()
|
||||
tmpfiles()
|
||||
nsswitch()
|
||||
os_release()
|
||||
python_alternatives()
|
||||
|
||||
r = subprocess.run(sys.argv[1:], check=False)
|
||||
sys.exit(r.returncode)
|
||||
112
runners/org.osbuild.ubuntu1804
Executable file
112
runners/org.osbuild.ubuntu1804
Executable file
|
|
@ -0,0 +1,112 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import array
|
||||
import json
|
||||
import shutil
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
# copied from remoteloop.py
|
||||
def load_fds(sock, msglen):
|
||||
fds = array.array("i") # Array of ints
|
||||
msg, ancdata, _, addr = sock.recvmsg(msglen, socket.CMSG_LEN(253 * fds.itemsize))
|
||||
for cmsg_level, cmsg_type, cmsg_data in ancdata:
|
||||
if (cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS):
|
||||
# Append data, ignoring any truncated integers at the end.
|
||||
fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)])
|
||||
return json.loads(msg), list(fds), addr
|
||||
|
||||
def ldconfig():
|
||||
# ld.so.conf must exist, or `ldconfig` throws a warning
|
||||
subprocess.run(["touch", "/etc/ld.so.conf"], check=True)
|
||||
subprocess.run(["ldconfig"], check=True)
|
||||
|
||||
|
||||
def sysusers():
|
||||
try:
|
||||
subprocess.run(["systemd-sysusers"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
|
||||
except subprocess.CalledProcessError as error:
|
||||
sys.stderr.write(error.stdout)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def update_ca_trust():
|
||||
if not shutil.which("update-ca-trust"):
|
||||
return
|
||||
|
||||
# generate /etc/pki/tls/certs/ca-bundle.crt
|
||||
os.makedirs("/etc/pki/ca-trust/extracted/pem")
|
||||
os.makedirs("/etc/pki/tls/certs")
|
||||
os.symlink("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", "/etc/pki/tls/certs/ca-bundle.crt")
|
||||
|
||||
# allow to fail, because it sometimes mysteriously does
|
||||
subprocess.run(["update-ca-trust", "extract"], check=False)
|
||||
|
||||
|
||||
def append_certs(cert_conf, dir_fd, parents=b""):
|
||||
for entry in os.scandir(f"/proc/self/fd/{dir_fd}".encode()):
|
||||
if entry.is_file():
|
||||
line = os.path.join(parents, entry.name)
|
||||
cert_conf.write(line)
|
||||
cert_conf.write(b"\n")
|
||||
elif entry.is_dir():
|
||||
append_certs(cert_conf,
|
||||
os.open(entry.name, os.O_DIRECTORY, dir_fd=dir_fd),
|
||||
os.path.join(parents, entry.name))
|
||||
|
||||
|
||||
def update_ca_certificates():
|
||||
if not shutil.which("update-ca-certificates"):
|
||||
return
|
||||
|
||||
# generate /etc/ssl/certs/ca-certificates.crt
|
||||
os.makedirs("/etc/ssl/certs")
|
||||
with open("/etc/ca-certificates.conf", "wb") as f:
|
||||
append_certs(f, os.open("/usr/share/ca-certificates", os.O_DIRECTORY))
|
||||
subprocess.run(["update-ca-certificates"], check=True)
|
||||
|
||||
|
||||
def tmpfiles():
|
||||
# Allow systemd-tmpfiles to return non-0. Some packages want to create
|
||||
# directories owned by users that are not set up with systemd-sysusers.
|
||||
subprocess.run(["systemd-tmpfiles", "--create"], check=False)
|
||||
|
||||
|
||||
def nsswitch():
|
||||
# the default behavior is fine, but using nss-resolve does not
|
||||
# necessarily work in a non-booted container, so make sure that
|
||||
# is not configured.
|
||||
try:
|
||||
os.remove("/etc/nsswitch.conf")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def setup_stdio():
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1)
|
||||
sock.connect("/run/osbuild/api/osbuild")
|
||||
req = {'method': 'setup-stdio'}
|
||||
sock.send(json.dumps(req).encode('utf-8'))
|
||||
msg, fds, _ = load_fds(sock, 1024)
|
||||
for io in ['stdin', 'stdout', 'stderr']:
|
||||
target = getattr(sys, io)
|
||||
source = fds[msg[io]]
|
||||
os.dup2(source, target.fileno())
|
||||
os.close(source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
setup_stdio()
|
||||
ldconfig()
|
||||
sysusers()
|
||||
update_ca_trust()
|
||||
update_ca_certificates()
|
||||
tmpfiles()
|
||||
nsswitch()
|
||||
|
||||
r = subprocess.run(sys.argv[1:], check=False)
|
||||
sys.exit(r.returncode)
|
||||
Loading…
Add table
Add a link
Reference in a new issue