69d87c27ff
Implement a phase for the `imageBuilderBuild` task that is provided by the `koji-image-builder` plugin, which schedules tasks to run with `image-builder`. This change is part of an accepted change proposal [1] for Fedora to use `koji-image-builder` to build (some of) its variants. [1]: https://fedoraproject.org/wiki/Changes/KojiLocalImageBuilder Signed-off-by: Simon de Vlieger <supakeen@redhat.com>
263 lines
9.4 KiB
Python
263 lines
9.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
from kobo.threads import ThreadPool
|
|
from kobo import shortcuts
|
|
from productmd.images import Image
|
|
|
|
from . import base
|
|
from .. import util
|
|
from ..linker import Linker
|
|
from ..wrappers import kojiwrapper
|
|
from .image_build import EXTENSIONS
|
|
from ..threading import TelemetryWorkerThread as WorkerThread
|
|
|
|
|
|
IMAGEBUILDEREXTENSIONS = [
|
|
("vagrant-libvirt", ["vagrant.libvirt.box"], "vagrant-libvirt.box"),
|
|
(
|
|
"vagrant-virtualbox",
|
|
["vagrant.virtualbox.box"],
|
|
"vagrant-virtualbox.box",
|
|
),
|
|
("container", ["oci.tar.xz"], "tar.xz"),
|
|
("wsl2", ["wsl"], "wsl"),
|
|
# .iso images can be of many types - boot, cd, dvd, live... -
|
|
# so 'boot' is just a default guess. 'iso' is not a valid
|
|
# productmd image type
|
|
("boot", [".iso"], "iso"),
|
|
]
|
|
|
|
|
|
class ImageBuilderPhase(
|
|
base.PhaseLoggerMixin, base.ImageConfigMixin, base.ConfigGuardedPhase
|
|
):
|
|
name = "imagebuilder"
|
|
|
|
def __init__(self, compose):
|
|
super(ImageBuilderPhase, self).__init__(compose)
|
|
self.pool = ThreadPool(logger=self.logger)
|
|
|
|
def _get_arches(self, image_conf, arches):
|
|
"""Get an intersection of arches in the config dict and the given ones."""
|
|
if "arches" in image_conf:
|
|
arches = set(image_conf["arches"]) & arches
|
|
return sorted(arches)
|
|
|
|
@staticmethod
|
|
def _get_repo_urls(compose, repos, arch="$basearch"):
|
|
"""
|
|
Get list of repos with resolved repo URLs. Preserve repos defined
|
|
as dicts.
|
|
"""
|
|
resolved_repos = []
|
|
|
|
for repo in repos:
|
|
repo = util.get_repo_url(compose, repo, arch=arch)
|
|
if repo is None:
|
|
raise RuntimeError("Failed to resolve repo URL for %s" % repo)
|
|
resolved_repos.append(repo)
|
|
|
|
return resolved_repos
|
|
|
|
def _get_repo(self, image_conf, variant):
|
|
"""
|
|
Get a list of repos. First included are those explicitly listed in
|
|
config, followed by by repo for current variant if it's not included in
|
|
the list already.
|
|
"""
|
|
repos = shortcuts.force_list(image_conf.get("repos", []))
|
|
|
|
if not variant.is_empty and variant.uid not in repos:
|
|
repos.append(variant.uid)
|
|
|
|
return ImageBuilderPhase._get_repo_urls(self.compose, repos, arch="$arch")
|
|
|
|
def run(self):
|
|
for variant in self.compose.get_variants():
|
|
arches = set([x for x in variant.arches if x != "src"])
|
|
|
|
for image_conf in self.get_config_block(variant):
|
|
build_arches = self._get_arches(image_conf, arches)
|
|
if not build_arches:
|
|
self.log_debug("skip: no arches")
|
|
continue
|
|
|
|
# these properties can be set per-image *or* as e.g.
|
|
# imagebuilder_release or global_release in the config
|
|
generics = {
|
|
"release": self.get_release(image_conf),
|
|
"target": self.get_config(image_conf, "target"),
|
|
"types": self.get_config(image_conf, "types"),
|
|
"seed": self.get_config(image_conf, "seed"),
|
|
"scratch": self.get_config(image_conf, "scratch"),
|
|
"version": self.get_version(image_conf),
|
|
}
|
|
|
|
repo = self._get_repo(image_conf, variant)
|
|
|
|
failable_arches = image_conf.pop("failable", [])
|
|
if failable_arches == ["*"]:
|
|
failable_arches = image_conf["arches"]
|
|
|
|
self.pool.add(RunImageBuilderThread(self.pool))
|
|
self.pool.queue_put(
|
|
(
|
|
self.compose,
|
|
variant,
|
|
image_conf,
|
|
build_arches,
|
|
generics,
|
|
repo,
|
|
failable_arches,
|
|
)
|
|
)
|
|
|
|
self.pool.start()
|
|
|
|
|
|
class RunImageBuilderThread(WorkerThread):
|
|
def process(self, item, num):
|
|
(compose, variant, config, arches, generics, repo, failable_arches) = item
|
|
self.failable_arches = []
|
|
# the Koji task as a whole can only fail if *all* arches are failable
|
|
can_task_fail = set(self.failable_arches).issuperset(set(arches))
|
|
self.num = num
|
|
with util.failable(
|
|
compose,
|
|
can_task_fail,
|
|
variant,
|
|
"*",
|
|
"imageBuilderBuild",
|
|
logger=self.pool._logger,
|
|
):
|
|
self.worker(compose, variant, config, arches, generics, repo)
|
|
|
|
def worker(self, compose, variant, config, arches, generics, repo):
|
|
msg = "imageBuilderBuild task for variant %s" % variant.uid
|
|
self.pool.log_info("[BEGIN] %s" % msg)
|
|
koji = kojiwrapper.KojiWrapper(compose)
|
|
koji.login()
|
|
|
|
opts = {}
|
|
opts["repos"] = repo
|
|
|
|
if generics.get("release"):
|
|
opts["release"] = generics["release"]
|
|
|
|
if generics.get("seed"):
|
|
opts["seed"] = generics["seed"]
|
|
|
|
if generics.get("scratch"):
|
|
opts["scratch"] = generics["scratch"]
|
|
|
|
if config.get("ostree"):
|
|
opts["ostree"] = config["ostree"]
|
|
|
|
if config.get("blueprint"):
|
|
opts["blueprint"] = config["blueprint"]
|
|
|
|
task_id = koji.koji_proxy.imageBuilderBuild(
|
|
generics["target"],
|
|
arches,
|
|
types=generics["types"],
|
|
name=config["name"],
|
|
version=generics["version"],
|
|
opts=opts,
|
|
)
|
|
|
|
koji.save_task_id(task_id)
|
|
|
|
# Wait for it to finish and capture the output into log file.
|
|
log_dir = os.path.join(compose.paths.log.topdir(), "imageBuilderBuild")
|
|
util.makedirs(log_dir)
|
|
log_file = os.path.join(
|
|
log_dir, "%s-%s-watch-task.log" % (variant.uid, self.num)
|
|
)
|
|
if koji.watch_task(task_id, log_file) != 0:
|
|
raise RuntimeError(
|
|
"imageBuilderBuild task failed: %s. See %s for details"
|
|
% (task_id, log_file)
|
|
)
|
|
|
|
# Refresh koji session which may have timed out while the task was
|
|
# running. Watching is done via a subprocess, so the session is
|
|
# inactive.
|
|
koji = kojiwrapper.KojiWrapper(compose)
|
|
|
|
linker = Linker(logger=self.pool._logger)
|
|
|
|
# Process all images in the build. There should be one for each
|
|
# architecture, but we don't verify that.
|
|
paths = koji.get_image_paths(task_id)
|
|
|
|
for arch, paths in paths.items():
|
|
for path in paths:
|
|
type_, format_ = _find_type_and_format(path)
|
|
if not format_:
|
|
# Path doesn't match any known type.
|
|
continue
|
|
|
|
# image_dir is absolute path to which the image should be copied.
|
|
# We also need the same path as relative to compose directory for
|
|
# including in the metadata.
|
|
if format_ == "iso":
|
|
# If the produced image is actually an ISO, it should go to
|
|
# iso/ subdirectory.
|
|
image_dir = compose.paths.compose.iso_dir(arch, variant)
|
|
rel_image_dir = compose.paths.compose.iso_dir(
|
|
arch, variant, relative=True
|
|
)
|
|
else:
|
|
image_dir = compose.paths.compose.image_dir(variant) % {
|
|
"arch": arch
|
|
}
|
|
rel_image_dir = compose.paths.compose.image_dir(
|
|
variant, relative=True
|
|
) % {"arch": arch}
|
|
util.makedirs(image_dir)
|
|
|
|
filename = os.path.basename(path)
|
|
|
|
image_dest = os.path.join(image_dir, filename)
|
|
|
|
src_file = compose.koji_downloader.get_file(path)
|
|
|
|
linker.link(src_file, image_dest, link_type=compose.conf["link_type"])
|
|
|
|
# Update image manifest
|
|
img = Image(compose.im)
|
|
|
|
# If user configured exact type, use it, otherwise try to
|
|
# figure it out based on the koji output.
|
|
img.type = config.get("manifest_type", type_)
|
|
img.format = format_
|
|
img.path = os.path.join(rel_image_dir, filename)
|
|
img.mtime = util.get_mtime(image_dest)
|
|
img.size = util.get_file_size(image_dest)
|
|
img.arch = arch
|
|
img.disc_number = 1 # We don't expect multiple disks
|
|
img.disc_count = 1
|
|
|
|
img.bootable = format_ == "iso"
|
|
img.subvariant = config.get("subvariant", variant.uid)
|
|
setattr(img, "can_fail", arch in self.failable_arches)
|
|
setattr(img, "deliverable", "imageBuilderBuild")
|
|
compose.im.add(variant=variant.uid, arch=arch, image=img)
|
|
|
|
self.pool.log_info("[DONE ] %s (task id: %s)" % (msg, task_id))
|
|
|
|
|
|
def _find_type_and_format(path):
|
|
# these are our image-builder-exclusive mappings for images whose extensions
|
|
# aren't quite the same as imagefactory. they come first as we
|
|
# want our oci.tar.xz mapping to win over the tar.xz one in
|
|
# EXTENSIONS
|
|
for type_, suffixes, format_ in IMAGEBUILDEREXTENSIONS:
|
|
if any(path.endswith(suffix) for suffix in suffixes):
|
|
return type_, format_
|
|
for type_, suffixes in EXTENSIONS.items():
|
|
for suffix in suffixes:
|
|
if path.endswith(suffix):
|
|
return type_, suffix
|
|
return None, None
|