fcf1442f71
When `link_type = "symlink"` is used, the packages are in fact symlinks to /mnt/koji. When graft points file is generated, the paths in this graft points file point to symlinks and therefore symlinks are copied into the generated ISO file instead of real files. In this commit, the code to generate the graft points file is changed so it resolves the symlink to real file stored on /mnt/koji. To make this code safer, it does such resolving only in case the symlink points outside of `compose.paths.compose.topdir()`. Therefore you can still generate ISO file with symlink pointing to file stored within the ISO file itself, although this is not done currently afaik. The main reason for this is to be able to generate ISO files even without hardlinks (which would need read-write access on /mnt/koji) and without copying all the packages from /mnt/koji to local storage. Signed-off-by: Jan Kaluza <jkaluza@redhat.com>
521 lines
20 KiB
Python
521 lines
20 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; version 2 of the License.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU Library General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, see <https://gnu.org/licenses/>.
|
|
|
|
|
|
import os
|
|
import random
|
|
import shutil
|
|
import stat
|
|
|
|
import productmd.treeinfo
|
|
from productmd.images import Image
|
|
from kobo.threads import ThreadPool, WorkerThread
|
|
from kobo.shortcuts import run, relative_path
|
|
from six.moves import shlex_quote
|
|
|
|
from pungi.wrappers import iso
|
|
from pungi.wrappers.createrepo import CreaterepoWrapper
|
|
from pungi.wrappers import kojiwrapper
|
|
from pungi.phases.base import PhaseBase, PhaseLoggerMixin
|
|
from pungi.util import (makedirs, get_volid, get_arch_variant_data, failable,
|
|
get_file_size, get_mtime)
|
|
from pungi.media_split import MediaSplitter, convert_media_size
|
|
from pungi.compose_metadata.discinfo import read_discinfo, write_discinfo
|
|
from pungi.runroot import Runroot
|
|
|
|
from .. import createiso
|
|
|
|
|
|
class CreateisoPhase(PhaseLoggerMixin, PhaseBase):
|
|
name = "createiso"
|
|
|
|
def __init__(self, compose, buildinstall_phase):
|
|
super(CreateisoPhase, self).__init__(compose)
|
|
self.pool = ThreadPool(logger=self.logger)
|
|
self.bi = buildinstall_phase
|
|
|
|
def _find_rpms(self, path):
|
|
"""Check if there are some RPMs in the path."""
|
|
for _, _, files in os.walk(path):
|
|
for fn in files:
|
|
if fn.endswith(".rpm"):
|
|
return True
|
|
return False
|
|
|
|
def _is_bootable(self, variant, arch):
|
|
if arch == "src":
|
|
return False
|
|
if variant.type != "variant":
|
|
return False
|
|
skip = get_arch_variant_data(self.compose.conf, "buildinstall_skip", arch, variant)
|
|
if skip == [True]:
|
|
# Buildinstall is skipped for this tree. Can't create a bootable ISO.
|
|
return False
|
|
return bool(self.compose.conf.get('buildinstall_method', ''))
|
|
|
|
def run(self):
|
|
symlink_isos_to = self.compose.conf.get("symlink_isos_to")
|
|
disc_type = self.compose.conf['disc_types'].get('dvd', 'dvd')
|
|
deliverables = []
|
|
|
|
commands = []
|
|
for variant in self.compose.get_variants(types=["variant", "layered-product", "optional"]):
|
|
if variant.is_empty:
|
|
continue
|
|
for arch in variant.arches + ["src"]:
|
|
skip_iso = get_arch_variant_data(self.compose.conf, "createiso_skip", arch, variant)
|
|
if skip_iso == [True]:
|
|
self.logger.info("Skipping createiso for %s.%s due to config option" % (variant, arch))
|
|
continue
|
|
|
|
volid = get_volid(self.compose, arch, variant, disc_type=disc_type)
|
|
os_tree = self.compose.paths.compose.os_tree(arch, variant)
|
|
|
|
iso_dir = self.compose.paths.compose.iso_dir(arch, variant, symlink_to=symlink_isos_to)
|
|
if not iso_dir:
|
|
continue
|
|
|
|
if not self._find_rpms(os_tree):
|
|
self.logger.warn("No RPMs found for %s.%s, skipping ISO"
|
|
% (variant.uid, arch))
|
|
continue
|
|
|
|
bootable = self._is_bootable(variant, arch)
|
|
|
|
if bootable and not self.bi.succeeded(variant, arch):
|
|
self.logger.warning(
|
|
'ISO should be bootable, but buildinstall failed. Skipping for %s.%s'
|
|
% (variant, arch))
|
|
continue
|
|
|
|
split_iso_data = split_iso(self.compose, arch, variant, no_split=bootable,
|
|
logger=self.logger)
|
|
disc_count = len(split_iso_data)
|
|
|
|
for disc_num, iso_data in enumerate(split_iso_data):
|
|
disc_num += 1
|
|
|
|
filename = self.compose.get_image_name(
|
|
arch, variant, disc_type=disc_type, disc_num=disc_num)
|
|
iso_path = self.compose.paths.compose.iso_path(
|
|
arch, variant, filename, symlink_to=symlink_isos_to)
|
|
if os.path.isfile(iso_path):
|
|
self.logger.warn("Skipping mkisofs, image already exists: %s" % iso_path)
|
|
continue
|
|
deliverables.append(iso_path)
|
|
|
|
graft_points = prepare_iso(self.compose, arch, variant,
|
|
disc_num=disc_num, disc_count=disc_count,
|
|
split_iso_data=iso_data)
|
|
|
|
cmd = {
|
|
"iso_path": iso_path,
|
|
"bootable": bootable,
|
|
"cmd": [],
|
|
"label": "", # currently not used
|
|
"disc_num": disc_num,
|
|
"disc_count": disc_count,
|
|
}
|
|
|
|
if os.path.islink(iso_dir):
|
|
cmd["mount"] = os.path.abspath(os.path.join(os.path.dirname(iso_dir),
|
|
os.readlink(iso_dir)))
|
|
|
|
opts = createiso.CreateIsoOpts(
|
|
output_dir=iso_dir,
|
|
iso_name=filename,
|
|
volid=volid,
|
|
graft_points=graft_points,
|
|
arch=arch,
|
|
supported=self.compose.supported,
|
|
hfs_compat=self.compose.conf["iso_hfs_ppc64le_compatible"],
|
|
)
|
|
|
|
if bootable:
|
|
opts = opts._replace(buildinstall_method=self.compose.conf['buildinstall_method'])
|
|
|
|
if self.compose.conf['create_jigdo']:
|
|
jigdo_dir = self.compose.paths.compose.jigdo_dir(arch, variant)
|
|
opts = opts._replace(jigdo_dir=jigdo_dir, os_tree=os_tree)
|
|
|
|
script_file = os.path.join(self.compose.paths.work.tmp_dir(arch, variant),
|
|
'createiso-%s.sh' % filename)
|
|
with open(script_file, 'w') as f:
|
|
createiso.write_script(opts, f)
|
|
cmd['cmd'] = ['bash', script_file]
|
|
commands.append((cmd, variant, arch))
|
|
|
|
if self.compose.notifier:
|
|
self.compose.notifier.send('createiso-targets', deliverables=deliverables)
|
|
|
|
for (cmd, variant, arch) in commands:
|
|
self.pool.add(CreateIsoThread(self.pool))
|
|
self.pool.queue_put((self.compose, cmd, variant, arch))
|
|
|
|
self.pool.start()
|
|
|
|
|
|
class CreateIsoThread(WorkerThread):
|
|
def fail(self, compose, cmd, variant, arch):
|
|
self.pool.log_error("CreateISO failed, removing ISO: %s" % cmd["iso_path"])
|
|
try:
|
|
# remove incomplete ISO
|
|
os.unlink(cmd["iso_path"])
|
|
# TODO: remove jigdo & template
|
|
except OSError:
|
|
pass
|
|
if compose.notifier:
|
|
compose.notifier.send('createiso-imagefail',
|
|
file=cmd['iso_path'],
|
|
arch=arch,
|
|
variant=str(variant))
|
|
|
|
def process(self, item, num):
|
|
compose, cmd, variant, arch = item
|
|
can_fail = compose.can_fail(variant, arch, 'iso')
|
|
with failable(compose, can_fail, variant, arch, 'iso', logger=self.pool._logger):
|
|
self.worker(compose, cmd, variant, arch, num)
|
|
|
|
def worker(self, compose, cmd, variant, arch, num):
|
|
mounts = [compose.topdir]
|
|
if "mount" in cmd:
|
|
mounts.append(cmd["mount"])
|
|
|
|
bootable = cmd['bootable']
|
|
log_file = compose.paths.log.log_file(
|
|
arch, "createiso-%s" % os.path.basename(cmd["iso_path"]))
|
|
|
|
msg = "Creating ISO (arch: %s, variant: %s): %s" % (
|
|
arch, variant, os.path.basename(cmd["iso_path"]))
|
|
self.pool.log_info("[BEGIN] %s" % msg)
|
|
|
|
try:
|
|
run_createiso_command(num, compose, bootable, arch,
|
|
cmd['cmd'], mounts, log_file)
|
|
except Exception:
|
|
self.fail(compose, cmd, variant, arch)
|
|
raise
|
|
|
|
add_iso_to_metadata(compose, variant, arch, cmd["iso_path"],
|
|
cmd["bootable"], cmd["disc_num"], cmd["disc_count"])
|
|
|
|
# Delete staging directory if present.
|
|
staging_dir = compose.paths.work.iso_staging_dir(
|
|
arch, variant, filename=os.path.basename(cmd["iso_path"])
|
|
)
|
|
if os.path.exists(staging_dir):
|
|
shutil.rmtree(staging_dir)
|
|
|
|
self.pool.log_info("[DONE ] %s" % msg)
|
|
if compose.notifier:
|
|
compose.notifier.send('createiso-imagedone',
|
|
file=cmd['iso_path'],
|
|
arch=arch,
|
|
variant=str(variant))
|
|
|
|
|
|
def add_iso_to_metadata(
|
|
compose,
|
|
variant,
|
|
arch,
|
|
iso_path,
|
|
bootable,
|
|
disc_num=1,
|
|
disc_count=1,
|
|
additional_variants=None,
|
|
):
|
|
img = Image(compose.im)
|
|
img.path = iso_path.replace(compose.paths.compose.topdir(), '').lstrip('/')
|
|
img.mtime = get_mtime(iso_path)
|
|
img.size = get_file_size(iso_path)
|
|
img.arch = arch
|
|
# XXX: HARDCODED
|
|
img.type = "dvd"
|
|
img.format = "iso"
|
|
img.disc_number = disc_num
|
|
img.disc_count = disc_count
|
|
img.bootable = bootable
|
|
img.subvariant = variant.uid
|
|
img.implant_md5 = iso.get_implanted_md5(iso_path, logger=compose._logger)
|
|
if additional_variants:
|
|
img.unified = True
|
|
img.additional_variants = additional_variants
|
|
setattr(img, 'can_fail', compose.can_fail(variant, arch, 'iso'))
|
|
setattr(img, 'deliverable', 'iso')
|
|
try:
|
|
img.volume_id = iso.get_volume_id(iso_path)
|
|
except RuntimeError:
|
|
pass
|
|
if arch == "src":
|
|
for variant_arch in variant.arches:
|
|
compose.im.add(variant.uid, variant_arch, img)
|
|
else:
|
|
compose.im.add(variant.uid, arch, img)
|
|
return img
|
|
|
|
|
|
def run_createiso_command(num, compose, bootable, arch, cmd, mounts,
|
|
log_file, with_jigdo=True):
|
|
packages = ["coreutils", "genisoimage", "isomd5sum"]
|
|
if with_jigdo and compose.conf['create_jigdo']:
|
|
packages.append('jigdo')
|
|
if bootable:
|
|
extra_packages = {
|
|
'lorax': ['lorax', 'which'],
|
|
'buildinstall': ['anaconda'],
|
|
}
|
|
packages.extend(extra_packages[compose.conf["buildinstall_method"]])
|
|
|
|
runroot = Runroot(compose, phase="createiso")
|
|
|
|
build_arch = arch
|
|
if runroot.runroot_method == "koji" and not bootable:
|
|
runroot_tag = compose.conf["runroot_tag"]
|
|
koji_wrapper = kojiwrapper.KojiWrapper(compose.conf["koji_profile"])
|
|
koji_proxy = koji_wrapper.koji_proxy
|
|
tag_info = koji_proxy.getTag(runroot_tag)
|
|
if not tag_info:
|
|
raise RuntimeError('Tag "%s" does not exist.' % runroot_tag)
|
|
tag_arches = tag_info["arches"].split(" ")
|
|
|
|
if "x86_64" in tag_arches:
|
|
# assign non-bootable images to x86_64 if possible
|
|
build_arch = "x86_64"
|
|
elif build_arch == "src":
|
|
# pick random arch from available runroot tag arches
|
|
build_arch = random.choice(tag_arches)
|
|
|
|
runroot.run(
|
|
cmd, log_file=log_file, arch=build_arch, packages=packages, mounts=mounts,
|
|
weight=compose.conf['runroot_weights'].get('createiso'))
|
|
|
|
|
|
def split_iso(compose, arch, variant, no_split=False, logger=None):
|
|
"""
|
|
Split contents of the os/ directory for given tree into chunks fitting on ISO.
|
|
|
|
All files from the directory are taken except for possible boot.iso image.
|
|
Files added in extra_files phase are put on all disks.
|
|
|
|
If `no_split` is set, we will pretend that the media is practically
|
|
infinite so that everything goes on single disc. A warning is printed if
|
|
the size is bigger than configured.
|
|
"""
|
|
if not logger:
|
|
logger = compose._logger
|
|
media_size = compose.conf['iso_size']
|
|
media_reserve = compose.conf['split_iso_reserve']
|
|
split_size = convert_media_size(media_size) - convert_media_size(media_reserve)
|
|
real_size = None if no_split else split_size
|
|
|
|
ms = MediaSplitter(real_size, compose, logger=logger)
|
|
|
|
os_tree = compose.paths.compose.os_tree(arch, variant)
|
|
extra_files_dir = compose.paths.work.extra_files_dir(arch, variant)
|
|
|
|
# scan extra files to mark them "sticky" -> they'll be on all media after split
|
|
extra_files = set(["media.repo"])
|
|
for root, dirs, files in os.walk(extra_files_dir):
|
|
for fn in files:
|
|
path = os.path.join(root, fn)
|
|
rel_path = relative_path(path, extra_files_dir.rstrip("/") + "/")
|
|
extra_files.add(rel_path)
|
|
|
|
packages = []
|
|
all_files = []
|
|
all_files_ignore = []
|
|
|
|
ti = productmd.treeinfo.TreeInfo()
|
|
ti.load(os.path.join(os_tree, ".treeinfo"))
|
|
boot_iso_rpath = ti.images.images.get(arch, {}).get("boot.iso", None)
|
|
if boot_iso_rpath:
|
|
all_files_ignore.append(boot_iso_rpath)
|
|
if all_files_ignore:
|
|
logger.debug("split_iso all_files_ignore = %s" % ", ".join(all_files_ignore))
|
|
|
|
for root, dirs, files in os.walk(os_tree):
|
|
for dn in dirs[:]:
|
|
repo_dir = os.path.join(root, dn)
|
|
if repo_dir == os.path.join(compose.paths.compose.repository(arch, variant), "repodata"):
|
|
dirs.remove(dn)
|
|
|
|
for fn in files:
|
|
path = os.path.join(root, fn)
|
|
rel_path = relative_path(path, os_tree.rstrip("/") + "/")
|
|
sticky = rel_path in extra_files
|
|
if rel_path in all_files_ignore:
|
|
logger.info("split_iso: Skipping %s" % rel_path)
|
|
continue
|
|
if root.startswith(compose.paths.compose.packages(arch, variant)):
|
|
packages.append((path, os.path.getsize(path), sticky))
|
|
else:
|
|
all_files.append((path, os.path.getsize(path), sticky))
|
|
|
|
for path, size, sticky in all_files + packages:
|
|
ms.add_file(path, size, sticky)
|
|
|
|
logger.debug('Splitting media for %s.%s:' % (variant.uid, arch))
|
|
result = ms.split()
|
|
if no_split and result[0]['size'] > split_size:
|
|
logger.warn('ISO for %s.%s does not fit on single media! '
|
|
'It is %s bytes too big. (Total size: %s B)'
|
|
% (variant.uid, arch,
|
|
result[0]['size'] - split_size,
|
|
result[0]['size']))
|
|
return result
|
|
|
|
|
|
def prepare_iso(compose, arch, variant, disc_num=1, disc_count=None, split_iso_data=None):
|
|
tree_dir = compose.paths.compose.os_tree(arch, variant)
|
|
filename = compose.get_image_name(arch, variant, disc_num=disc_num)
|
|
iso_dir = compose.paths.work.iso_dir(arch, filename)
|
|
|
|
# modify treeinfo
|
|
ti_path = os.path.join(tree_dir, ".treeinfo")
|
|
ti = load_and_tweak_treeinfo(ti_path, disc_num, disc_count)
|
|
|
|
copy_boot_images(tree_dir, iso_dir)
|
|
|
|
if disc_count > 1:
|
|
# remove repodata/repomd.xml from checksums, create a new one later
|
|
if "repodata/repomd.xml" in ti.checksums.checksums:
|
|
del ti.checksums.checksums["repodata/repomd.xml"]
|
|
|
|
# rebuild repodata
|
|
createrepo_c = compose.conf["createrepo_c"]
|
|
createrepo_checksum = compose.conf["createrepo_checksum"]
|
|
repo = CreaterepoWrapper(createrepo_c=createrepo_c)
|
|
|
|
file_list = "%s-file-list" % iso_dir
|
|
packages_dir = compose.paths.compose.packages(arch, variant)
|
|
file_list_content = []
|
|
for i in split_iso_data["files"]:
|
|
if not i.endswith(".rpm"):
|
|
continue
|
|
if not i.startswith(packages_dir):
|
|
continue
|
|
rel_path = relative_path(i, tree_dir.rstrip("/") + "/")
|
|
file_list_content.append(rel_path)
|
|
|
|
if file_list_content:
|
|
# write modified repodata only if there are packages available
|
|
run("cp -a %s/repodata %s/" % (shlex_quote(tree_dir), shlex_quote(iso_dir)))
|
|
with open(file_list, "w") as f:
|
|
f.write("\n".join(file_list_content))
|
|
cmd = repo.get_createrepo_cmd(
|
|
tree_dir,
|
|
update=True,
|
|
database=True,
|
|
skip_stat=True,
|
|
pkglist=file_list,
|
|
outputdir=iso_dir,
|
|
workers=compose.conf["createrepo_num_workers"],
|
|
checksum=createrepo_checksum,
|
|
)
|
|
run(cmd)
|
|
# add repodata/repomd.xml back to checksums
|
|
ti.checksums.add("repodata/repomd.xml", createrepo_checksum, root_dir=iso_dir)
|
|
|
|
new_ti_path = os.path.join(iso_dir, ".treeinfo")
|
|
ti.dump(new_ti_path)
|
|
|
|
# modify discinfo
|
|
di_path = os.path.join(tree_dir, ".discinfo")
|
|
data = read_discinfo(di_path)
|
|
data["disc_numbers"] = [disc_num]
|
|
new_di_path = os.path.join(iso_dir, ".discinfo")
|
|
write_discinfo(new_di_path, **data)
|
|
|
|
if not disc_count or disc_count == 1:
|
|
data = iso.get_graft_points(compose, [tree_dir, iso_dir])
|
|
else:
|
|
data = iso.get_graft_points(compose, [iso._paths_from_list(tree_dir, split_iso_data["files"]), iso_dir])
|
|
|
|
if compose.conf["createiso_break_hardlinks"]:
|
|
compose.log_debug(
|
|
"Breaking hardlinks for ISO %s for %s.%s" % (filename, variant, arch)
|
|
)
|
|
break_hardlinks(
|
|
data, compose.paths.work.iso_staging_dir(arch, variant, filename)
|
|
)
|
|
# Create hardlinks for files with duplicate contents.
|
|
compose.log_debug(
|
|
"Creating hardlinks for ISO %s for %s.%s" % (filename, variant, arch)
|
|
)
|
|
create_hardlinks(
|
|
compose.paths.work.iso_staging_dir(arch, variant, filename),
|
|
log_file=compose.paths.log.log_file(arch, "iso-hardlink-%s.log" % variant.uid),
|
|
)
|
|
|
|
# TODO: /content /graft-points
|
|
gp = "%s-graft-points" % iso_dir
|
|
iso.write_graft_points(gp, data, exclude=["*/lost+found", "*/boot.iso"])
|
|
return gp
|
|
|
|
|
|
def load_and_tweak_treeinfo(ti_path, disc_num=1, disc_count=1):
|
|
"""Treeinfo on the media should not contain any reference to boot.iso and
|
|
it should also have a valid [media] section.
|
|
"""
|
|
ti = productmd.treeinfo.TreeInfo()
|
|
ti.load(ti_path)
|
|
ti.media.totaldiscs = disc_count or 1
|
|
ti.media.discnum = disc_num
|
|
|
|
# remove boot.iso from all sections
|
|
paths = set()
|
|
for platform in ti.images.images:
|
|
if "boot.iso" in ti.images.images[platform]:
|
|
paths.add(ti.images.images[platform].pop("boot.iso"))
|
|
|
|
# remove boot.iso from checksums
|
|
for i in paths:
|
|
if i in ti.checksums.checksums.keys():
|
|
del ti.checksums.checksums[i]
|
|
|
|
return ti
|
|
|
|
|
|
def copy_boot_images(src, dest):
|
|
"""When mkisofs is called it tries to modify isolinux/isolinux.bin and
|
|
images/boot.img. Therefore we need to make copies of them.
|
|
"""
|
|
for i in ("isolinux/isolinux.bin", "images/boot.img"):
|
|
src_path = os.path.join(src, i)
|
|
dst_path = os.path.join(dest, i)
|
|
if os.path.exists(src_path):
|
|
makedirs(os.path.dirname(dst_path))
|
|
shutil.copy2(src_path, dst_path)
|
|
|
|
|
|
def break_hardlinks(graft_points, staging_dir):
|
|
"""Iterate over graft points and copy any file that has more than 1
|
|
hardlink into the staging directory. Replace the entry in the dict.
|
|
"""
|
|
for f in graft_points:
|
|
info = os.stat(graft_points[f])
|
|
if stat.S_ISREG(info.st_mode) and info.st_nlink > 1:
|
|
dest_path = os.path.join(staging_dir, graft_points[f].lstrip("/"))
|
|
makedirs(os.path.dirname(dest_path))
|
|
shutil.copy2(graft_points[f], dest_path)
|
|
graft_points[f] = dest_path
|
|
|
|
|
|
def create_hardlinks(staging_dir, log_file):
|
|
"""Create hardlinks within the staging directory.
|
|
Should happen after break_hardlinks()
|
|
"""
|
|
cmd = ["/usr/sbin/hardlink", "-c", "-vv", staging_dir]
|
|
run(cmd, logfile=log_file, show_cmd=True)
|