f9a6c8418f
The schema is written in Python to reduce duplication. When configuration is loaded, the validation checks if it's correct and fills in default values. There is a custom extension to the schema to report deprecated options. The config dependencies are implemented as a separate pass. While it's technically possible to express the dependencies in the schema itself, the error messages are not very helpful and it makes the schema much harder to read. Phases no longer define `config_options`. New options should be added to the schema. Since the default values are populated automatically during validation, there is no need to duplicate them into the code. The `pungi-config-validate` script is updated to use the schema and report errors even for deeply nested fields. The dependencies are updated: pungi now depends on `python-jsonschema` (which is already available in Fedora). Signed-off-by: Lubomír Sedlář <lsedlar@redhat.com>
409 lines
16 KiB
Python
409 lines
16 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
|
|
|
|
|
|
import errno
|
|
import os
|
|
import time
|
|
import pipes
|
|
import tempfile
|
|
import shutil
|
|
import re
|
|
|
|
from kobo.threads import ThreadPool, WorkerThread
|
|
from kobo.shortcuts import run
|
|
from productmd.images import Image
|
|
|
|
from pungi.arch import get_valid_arches
|
|
from pungi.util import get_buildroot_rpms, get_volid, get_arch_variant_data
|
|
from pungi.util import get_file_size, get_mtime, failable
|
|
from pungi.wrappers.lorax import LoraxWrapper
|
|
from pungi.wrappers.kojiwrapper import KojiWrapper
|
|
from pungi.wrappers.iso import IsoWrapper
|
|
from pungi.wrappers.scm import get_file_from_scm
|
|
from pungi.phases.base import PhaseBase
|
|
|
|
|
|
class BuildinstallPhase(PhaseBase):
|
|
name = "buildinstall"
|
|
|
|
def __init__(self, compose):
|
|
PhaseBase.__init__(self, compose)
|
|
self.pool = ThreadPool(logger=self.compose._logger)
|
|
|
|
def skip(self):
|
|
if PhaseBase.skip(self):
|
|
return True
|
|
if not self.compose.conf.get("bootable"):
|
|
msg = "Not a bootable product. Skipping buildinstall."
|
|
self.compose.log_debug(msg)
|
|
return True
|
|
return False
|
|
|
|
def _get_lorax_cmd(self, repo_baseurl, output_dir, variant, arch, buildarch, volid):
|
|
noupgrade = True
|
|
bugurl = None
|
|
nomacboot = True
|
|
for data in get_arch_variant_data(self.compose.conf, 'lorax_options', arch, variant):
|
|
if not data.get('noupgrade', True):
|
|
noupgrade = False
|
|
if data.get('bugurl'):
|
|
bugurl = data.get('bugurl')
|
|
if not data.get('nomacboot', True):
|
|
nomacboot = False
|
|
output_dir = os.path.join(output_dir, variant.uid)
|
|
lorax = LoraxWrapper()
|
|
lorax_cmd = lorax.get_lorax_cmd(self.compose.conf["release_name"],
|
|
self.compose.conf["release_version"],
|
|
self.compose.conf["release_version"],
|
|
repo_baseurl,
|
|
output_dir,
|
|
variant=variant.uid,
|
|
buildinstallpackages=variant.buildinstallpackages,
|
|
is_final=self.compose.supported,
|
|
buildarch=buildarch,
|
|
volid=volid,
|
|
nomacboot=nomacboot,
|
|
bugurl=bugurl,
|
|
noupgrade=noupgrade)
|
|
return 'rm -rf %s && %s' % (pipes.quote(output_dir),
|
|
' '.join([pipes.quote(x) for x in lorax_cmd]))
|
|
|
|
def run(self):
|
|
lorax = LoraxWrapper()
|
|
product = self.compose.conf["release_name"]
|
|
version = self.compose.conf["release_version"]
|
|
release = self.compose.conf["release_version"]
|
|
buildinstall_method = self.compose.conf["buildinstall_method"]
|
|
disc_type = self.compose.conf['disc_types'].get('dvd', 'dvd')
|
|
|
|
for arch in self.compose.get_arches():
|
|
commands = []
|
|
|
|
repo_baseurl = self.compose.paths.work.arch_repo(arch)
|
|
output_dir = self.compose.paths.work.buildinstall_dir(arch)
|
|
buildarch = get_valid_arches(arch)[0]
|
|
|
|
if buildinstall_method == "lorax":
|
|
for variant in self.compose.get_variants(arch=arch, types=['variant']):
|
|
if variant.is_empty:
|
|
continue
|
|
volid = get_volid(self.compose, arch, variant=variant, disc_type=disc_type)
|
|
commands.append(
|
|
(variant,
|
|
self._get_lorax_cmd(repo_baseurl, output_dir, variant, arch, buildarch, volid))
|
|
)
|
|
elif buildinstall_method == "buildinstall":
|
|
volid = get_volid(self.compose, arch, disc_type=disc_type)
|
|
commands.append(
|
|
(None,
|
|
lorax.get_buildinstall_cmd(product,
|
|
version,
|
|
release,
|
|
repo_baseurl,
|
|
output_dir,
|
|
is_final=self.compose.supported,
|
|
buildarch=buildarch,
|
|
volid=volid))
|
|
)
|
|
else:
|
|
raise ValueError("Unsupported buildinstall method: %s" % buildinstall_method)
|
|
|
|
for (variant, cmd) in commands:
|
|
self.pool.add(BuildinstallThread(self.pool))
|
|
self.pool.queue_put((self.compose, arch, variant, cmd))
|
|
|
|
self.pool.start()
|
|
|
|
def copy_files(self):
|
|
buildinstall_method = self.compose.conf["buildinstall_method"]
|
|
disc_type = self.compose.conf['disc_types'].get('dvd', 'dvd')
|
|
|
|
# copy buildinstall files to the 'os' dir
|
|
kickstart_file = get_kickstart_file(self.compose)
|
|
for arch in self.compose.get_arches():
|
|
for variant in self.compose.get_variants(arch=arch, types=["self", "variant"]):
|
|
if variant.is_empty:
|
|
continue
|
|
|
|
buildinstall_dir = self.compose.paths.work.buildinstall_dir(arch)
|
|
|
|
# Lorax runs per-variant, so we need to tweak the source path
|
|
# to include variant.
|
|
if buildinstall_method == 'lorax':
|
|
buildinstall_dir = os.path.join(buildinstall_dir, variant.uid)
|
|
|
|
if not os.path.isdir(buildinstall_dir) or not os.listdir(buildinstall_dir):
|
|
continue
|
|
|
|
os_tree = self.compose.paths.compose.os_tree(arch, variant)
|
|
# TODO: label is not used
|
|
label = ""
|
|
volid = get_volid(self.compose, arch, variant, escape_spaces=False, disc_type=disc_type)
|
|
can_fail = self.compose.can_fail(variant, arch, 'buildinstall')
|
|
with failable(self.compose, can_fail, variant, arch, 'buildinstall'):
|
|
tweak_buildinstall(buildinstall_dir, os_tree, arch, variant.uid, label, volid, kickstart_file)
|
|
link_boot_iso(self.compose, arch, variant, can_fail)
|
|
|
|
|
|
def get_kickstart_file(compose):
|
|
scm_dict = compose.conf.get("buildinstall_kickstart")
|
|
if not scm_dict:
|
|
compose.log_debug("Path to ks.cfg (buildinstall_kickstart) not specified.")
|
|
return
|
|
|
|
msg = "Getting ks.cfg"
|
|
kickstart_path = os.path.join(compose.paths.work.topdir(arch="global"), "ks.cfg")
|
|
if os.path.exists(kickstart_path):
|
|
compose.log_warn("[SKIP ] %s" % msg)
|
|
return kickstart_path
|
|
|
|
compose.log_info("[BEGIN] %s" % msg)
|
|
if isinstance(scm_dict, dict):
|
|
kickstart_name = os.path.basename(scm_dict["file"])
|
|
if scm_dict["scm"] == "file":
|
|
scm_dict["file"] = os.path.join(compose.config_dir, scm_dict["file"])
|
|
else:
|
|
kickstart_name = os.path.basename(scm_dict)
|
|
scm_dict = os.path.join(compose.config_dir, scm_dict)
|
|
|
|
tmp_dir = tempfile.mkdtemp(prefix="buildinstall_kickstart_")
|
|
get_file_from_scm(scm_dict, tmp_dir, logger=compose._logger)
|
|
src = os.path.join(tmp_dir, kickstart_name)
|
|
shutil.copy2(src, kickstart_path)
|
|
compose.log_info("[DONE ] %s" % msg)
|
|
return kickstart_path
|
|
|
|
|
|
# HACK: this is a hack!
|
|
# * it's quite trivial to replace volids
|
|
# * it's not easy to replace menu titles
|
|
# * we probably need to get this into lorax
|
|
def tweak_buildinstall(src, dst, arch, variant, label, volid, kickstart_file=None):
|
|
volid_escaped = volid.replace(" ", r"\x20").replace("\\", "\\\\")
|
|
volid_escaped_2 = volid_escaped.replace("\\", "\\\\")
|
|
tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall_")
|
|
|
|
# verify src
|
|
if not os.path.isdir(src):
|
|
raise OSError(errno.ENOENT, "Directory does not exist: %s" % src)
|
|
|
|
# create dst
|
|
try:
|
|
os.makedirs(dst)
|
|
except OSError as ex:
|
|
if ex.errno != errno.EEXIST:
|
|
raise
|
|
|
|
# copy src to temp
|
|
# TODO: place temp on the same device as buildinstall dir so we can hardlink
|
|
cmd = "cp -av --remove-destination %s/* %s/" % (pipes.quote(src), pipes.quote(tmp_dir))
|
|
run(cmd)
|
|
|
|
# tweak configs
|
|
configs = [
|
|
"isolinux/isolinux.cfg",
|
|
"etc/yaboot.conf",
|
|
"ppc/ppc64/yaboot.conf",
|
|
"EFI/BOOT/BOOTX64.conf",
|
|
"EFI/BOOT/grub.cfg",
|
|
]
|
|
for config in configs:
|
|
config_path = os.path.join(tmp_dir, config)
|
|
if not os.path.exists(config_path):
|
|
continue
|
|
|
|
data = open(config_path, "r").read()
|
|
os.unlink(config_path) # break hadlink by removing file writing a new one
|
|
|
|
new_volid = volid_escaped
|
|
if "yaboot" in config:
|
|
# double-escape volid in yaboot.conf
|
|
new_volid = volid_escaped_2
|
|
|
|
ks = ""
|
|
if kickstart_file:
|
|
shutil.copy2(kickstart_file, os.path.join(dst, "ks.cfg"))
|
|
ks = " ks=hd:LABEL=%s:/ks.cfg" % new_volid
|
|
|
|
# pre-f18
|
|
data = re.sub(r":CDLABEL=[^ \n]*", r":CDLABEL=%s%s" % (new_volid, ks), data)
|
|
# f18+
|
|
data = re.sub(r":LABEL=[^ \n]*", r":LABEL=%s%s" % (new_volid, ks), data)
|
|
data = re.sub(r"(search .* -l) '[^'\n]*'", r"\1 '%s'" % volid, data)
|
|
|
|
open(config_path, "w").write(data)
|
|
|
|
images = [
|
|
os.path.join(tmp_dir, "images", "efiboot.img"),
|
|
]
|
|
for image in images:
|
|
if not os.path.isfile(image):
|
|
continue
|
|
mount_tmp_dir = tempfile.mkdtemp(prefix="tweak_buildinstall")
|
|
cmd = ["mount", "-o", "loop", image, mount_tmp_dir]
|
|
run(cmd)
|
|
|
|
for config in configs:
|
|
config_path = os.path.join(tmp_dir, config)
|
|
config_in_image = os.path.join(mount_tmp_dir, config)
|
|
|
|
if os.path.isfile(config_in_image):
|
|
cmd = ["cp", "-v", "--remove-destination", config_path, config_in_image]
|
|
run(cmd)
|
|
|
|
cmd = ["umount", mount_tmp_dir]
|
|
run(cmd)
|
|
shutil.rmtree(mount_tmp_dir)
|
|
|
|
# HACK: make buildinstall files world readable
|
|
run("chmod -R a+rX %s" % pipes.quote(tmp_dir))
|
|
|
|
# copy temp to dst
|
|
cmd = "cp -av --remove-destination %s/* %s/" % (pipes.quote(tmp_dir), pipes.quote(dst))
|
|
run(cmd)
|
|
|
|
shutil.rmtree(tmp_dir)
|
|
|
|
|
|
def link_boot_iso(compose, arch, variant, can_fail):
|
|
if arch == "src":
|
|
return
|
|
|
|
disc_type = compose.conf['disc_types'].get('boot', 'boot')
|
|
|
|
symlink_isos_to = compose.conf.get("symlink_isos_to")
|
|
os_tree = compose.paths.compose.os_tree(arch, variant)
|
|
# TODO: find in treeinfo?
|
|
boot_iso_path = os.path.join(os_tree, "images", "boot.iso")
|
|
if not os.path.isfile(boot_iso_path):
|
|
return
|
|
|
|
msg = "Linking boot.iso (arch: %s, variant: %s)" % (arch, variant)
|
|
filename = compose.get_image_name(arch, variant, disc_type=disc_type,
|
|
disc_num=None, suffix=".iso")
|
|
new_boot_iso_path = compose.paths.compose.iso_path(arch, variant, filename,
|
|
symlink_to=symlink_isos_to)
|
|
new_boot_iso_relative_path = compose.paths.compose.iso_path(arch,
|
|
variant,
|
|
filename,
|
|
relative=True)
|
|
if os.path.exists(new_boot_iso_path):
|
|
# TODO: log
|
|
compose.log_warning("[SKIP ] %s" % msg)
|
|
return
|
|
|
|
compose.log_info("[BEGIN] %s" % msg)
|
|
# Try to hardlink, and copy if that fails
|
|
try:
|
|
os.link(boot_iso_path, new_boot_iso_path)
|
|
except OSError:
|
|
shutil.copy2(boot_iso_path, new_boot_iso_path)
|
|
|
|
iso = IsoWrapper()
|
|
implant_md5 = iso.get_implanted_md5(new_boot_iso_path)
|
|
iso_name = os.path.basename(new_boot_iso_path)
|
|
iso_dir = os.path.dirname(new_boot_iso_path)
|
|
|
|
# create iso manifest
|
|
run(iso.get_manifest_cmd(iso_name), workdir=iso_dir)
|
|
|
|
img = Image(compose.im)
|
|
img.path = new_boot_iso_relative_path
|
|
img.mtime = get_mtime(new_boot_iso_path)
|
|
img.size = get_file_size(new_boot_iso_path)
|
|
img.arch = arch
|
|
img.type = "boot"
|
|
img.format = "iso"
|
|
img.disc_number = 1
|
|
img.disc_count = 1
|
|
img.bootable = True
|
|
img.subvariant = variant.name
|
|
img.implant_md5 = implant_md5
|
|
setattr(img, 'can_fail', can_fail)
|
|
setattr(img, 'deliverable', 'buildinstall')
|
|
try:
|
|
img.volume_id = iso.get_volume_id(new_boot_iso_path)
|
|
except RuntimeError:
|
|
pass
|
|
compose.im.add(variant.uid, arch, img)
|
|
compose.log_info("[DONE ] %s" % msg)
|
|
|
|
|
|
class BuildinstallThread(WorkerThread):
|
|
def process(self, item, num):
|
|
# The variant is None unless lorax is used as buildinstall method.
|
|
compose, arch, variant, cmd = item
|
|
can_fail = compose.can_fail(variant, arch, 'buildinstall')
|
|
with failable(compose, can_fail, variant, arch, 'buildinstall'):
|
|
self.worker(compose, arch, variant, cmd, num)
|
|
|
|
def worker(self, compose, arch, variant, cmd, num):
|
|
runroot = compose.conf["runroot"]
|
|
buildinstall_method = compose.conf["buildinstall_method"]
|
|
log_filename = ('buildinstall-%s' % variant.uid) if variant else 'buildinstall'
|
|
log_file = compose.paths.log.log_file(arch, log_filename)
|
|
|
|
msg = "Running buildinstall for arch %s, variant %s" % (arch, variant)
|
|
|
|
output_dir = compose.paths.work.buildinstall_dir(arch)
|
|
if variant:
|
|
output_dir = os.path.join(output_dir, variant.uid)
|
|
|
|
if os.path.isdir(output_dir) and os.listdir(output_dir):
|
|
# output dir is *not* empty -> SKIP
|
|
self.pool.log_warning(
|
|
'[SKIP ] Buildinstall for arch %s, variant %s' % (arch, variant))
|
|
return
|
|
|
|
self.pool.log_info("[BEGIN] %s" % msg)
|
|
|
|
task_id = None
|
|
if runroot:
|
|
# run in a koji build root
|
|
packages = ["strace"]
|
|
if buildinstall_method == "lorax":
|
|
packages += ["lorax"]
|
|
elif buildinstall_method == "buildinstall":
|
|
packages += ["anaconda"]
|
|
runroot_channel = compose.conf.get("runroot_channel")
|
|
runroot_tag = compose.conf["runroot_tag"]
|
|
|
|
koji_wrapper = KojiWrapper(compose.conf["koji_profile"])
|
|
koji_cmd = koji_wrapper.get_runroot_cmd(runroot_tag, arch, cmd,
|
|
channel=runroot_channel,
|
|
use_shell=True, task_id=True,
|
|
packages=packages, mounts=[compose.topdir])
|
|
|
|
# avoid race conditions?
|
|
# Kerberos authentication failed: Permission denied in replay cache code (-1765328215)
|
|
time.sleep(num * 3)
|
|
|
|
output = koji_wrapper.run_runroot_cmd(koji_cmd, log_file=log_file)
|
|
if output["retcode"] != 0:
|
|
raise RuntimeError("Runroot task failed: %s. See %s for more details."
|
|
% (output["task_id"], log_file))
|
|
task_id = output["task_id"]
|
|
|
|
else:
|
|
# run locally
|
|
run(cmd, show_cmd=True, logfile=log_file)
|
|
|
|
log_file = compose.paths.log.log_file(arch, log_filename + '-RPMs')
|
|
rpms = get_buildroot_rpms(compose, task_id)
|
|
open(log_file, "w").write("\n".join(rpms))
|
|
|
|
self.pool.log_info("[DONE ] %s" % msg)
|