particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adding initial Driver Update Disk building support

Browse source
This commit is contained in:
djuarezg 2022-01-14 17:33:22 +01:00 • committed by Tomas Kopecek
parent 0f1df4035a
commit aa26a0a8c2
4 changed files with 482 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

49
docs/source/plugins.rst
View file

@ -281,3 +281,52 @@ Selecting other than default kiwi profile can be done by ``--kiwi-profile``
option. Similarly to other image tasks, alternative architecture failures can be
ignored for successful build by ``--can-fail`` option. ``--arch`` can be used to
limit build tag architectures.
Driver Update Disks building
===========================
**This is just a tech-preview. API/usage can drastically change in upcoming
releases**
Plugin for creating Driver Update Disks with ``mkisofs``.
All three parts (cli/hub/builder) needs to be installed. There is currently no
configuration except allowing the plugins (name is 'dud' for all components).
Builders have to be part of ``image`` channel and don't need to have any
specific library installed (mkisofs invocation/usage is only in buildroots not on
builder itself)
Buildtag needs to be configured by adding special group ``dud`` which should contain
the following packages:
.. code-block:: shell
$ koji add-group dud-build-tag dud
$ koji add-group-pkg dud-build-tag dud genisoimage
$ koji add-group-pkg dud-build-tag dud createrepo_c
$ koji add-group-pkg dud-build-tag dud dnf
$ koji add-group-pkg dud-build-tag dud dnf-plugins-core
Another thing we need to ensure is that we're building in chroot and not in
container.
.. code-block:: shell
$ koji edit-tag dud-build-tag -x mock.new_chroot=False
Calling the build itself is a matter of simple CLI call:
.. code-block: shell
$ koji dud-build dud-build-tag --scmurl=git+https://my.git/image-descriptions#master myamazingdud 1 package1 package2
The command options allows to bring all the package dependencies into the DUD
ISO with ``--alldeps``. ``--scmurl`` allows to include non-RPM related content
inside the produced ISO.
Similarly to other image tasks, alternative architecture failures can be
ignored for successful build by ``--can-fail`` option. ``--arch`` can be used
to limit build tag architectures.

319
plugins/builder/dud.py Normal file
View file

@ -0,0 +1,319 @@
import glob
import json
import os
import rpm
import xml.dom.minidom
from fnmatch import fnmatch
import koji
import time
import re
from koji.util import joinpath, to_list
from koji.tasks import ServerExit
from __main__ import BaseBuildTask, BuildImageTask, BuildRoot, SCM
# /usr/lib/koji-builder-plugins/
class DudBuildTask(BuildImageTask):
Methods = ['dudBuild']
_taskWeight = 1.0
def handler(self, dud_name, dud_version, arches, target, pkg_list, opts=None):
target_info = self.session.getBuildTarget(target, strict=True)
build_tag = target_info['build_tag']
repo_info = self.getRepo(build_tag)
# check requested arches against build tag
buildconfig = self.session.getBuildConfig(build_tag)
if not buildconfig['arches']:
raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
if arches:
for arch in arches:
if koji.canonArch(arch) not in tag_archlist:
raise koji.BuildError("Invalid arch for build tag: %s" % arch)
else:
arches = tag_archlist
if not opts:
opts = {}
if not opts.get('scratch'):
opts['scratch'] = False
if not opts.get('alldeps'):
opts['alldeps'] = False
if not opts.get('scmurl'):
opts['scmurl'] = None
if not opts.get('optional_arches'):
opts['optional_arches'] = []
self.opts = opts
name, version, release = dud_name, dud_version, None
bld_info = None
if '-' in version:
raise koji.ApplianceError('The Version may not have a hyphen')
if not opts['scratch']:
bld_info = self.initImageBuild(name, version, release, target_info, opts)
release = bld_info['release']
elif not release:
release = self.session.getNextRelease({'name': name, 'version': version})
try:
subtasks = {}
canfail = []
self.logger.info("Spawning jobs for image arches: %r" % (arches))
for arch in arches:
subtasks[arch] = self.session.host.subtask(
method='createDudIso',
arglist=[name, version, release, arch,
target_info, build_tag, repo_info,
pkg_list, opts], label=arch,
parent=self.id, arch=arch)
if arch in self.opts['optional_arches']:
canfail.append(subtasks[arch])
self.logger.info("Got image subtasks: %r" % (subtasks))
self.logger.info("Waiting on image subtasks (%s can fail)..." % canfail)
results = self.wait(to_list(subtasks.values()), all=True, failany=True, canfail=canfail)
# if everything failed, fail even if all subtasks are in canfail
self.logger.info('subtask results: %r', results)
all_failed = True
for result in results.values():
if not isinstance(result, dict) or 'faultCode' not in result:
all_failed = False
break
if all_failed:
raise koji.GenericError("all subtasks failed")
# determine ignored arch failures
ignored_arches = set()
for arch in arches:
if arch in self.opts['optional_arches']:
task_id = subtasks[arch]
result = results[task_id]
if isinstance(result, dict) and 'faultCode' in result:
ignored_arches.add(arch)
self.logger.info('Image Results for hub: %s' % results)
results = {str(k): v for k, v in results.items()}
if opts['scratch']:
self.session.host.moveImageBuildToScratch(self.id, results)
else:
self.session.host.completeImageBuild(self.id, bld_info['id'], results)
except (SystemExit, ServerExit, KeyboardInterrupt):
# we do not trap these
raise
except Exception:
if not opts['scratch']:
if bld_info:
self.session.host.failBuild(self.id, bld_info['id'])
raise
# tag it
if not opts['scratch'] and not opts.get('skip_tag'):
tag_task_id = self.session.host.subtask(method='tagBuild',
arglist=[target_info['dest_tag'],
bld_info['id'], False, None, True],
label='tag', parent=self.id, arch='noarch')
self.wait(tag_task_id)
# report results
report = ''
if opts['scratch']:
respath = ', '.join(
[os.path.join(koji.pathinfo.work(),
koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()])
report += 'Scratch '
else:
respath = koji.pathinfo.imagebuild(bld_info)
report += 'image build results in: %s' % respath
return report
class DudCreateImageTask(BaseBuildTask):
Methods = ['createDudIso']
_taskWeight = 1.0
def getImagePackagesFromPath(self, path):
"""
Read RPM header information from the yum cache available in the
given path. Returns a list of dictionaries for each RPM included.
"""
found = False
hdrlist = {}
# For non scratch builds this is a must or it will not work
fields = ['name', 'version', 'release', 'epoch', 'arch',
'buildtime', 'sigmd5']
for root, dirs, files in os.walk(path):
for f in files:
if fnmatch(f, '*.rpm'):
pkgfile = os.path.join(root, f)
hdr = koji.get_header_fields(pkgfile, fields)
hdr['size'] = os.path.getsize(pkgfile)
hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
del hdr['sigmd5']
hdrlist[os.path.basename(pkgfile)] = hdr
found = True
if not found:
raise koji.LiveCDError('No rpms found in root dir!')
return list(hdrlist.values())
def handler(self, dud_name, dud_version, dud_release, arch,
target_info, build_tag, repo_info,
pkg_list, opts=None):
self.opts = opts
self.logger.info("Running my dud task...")
build_tag = target_info['build_tag']
broot = BuildRoot(self.session, self.options,
tag=build_tag,
arch=arch,
task_id=self.id,
repo_id=repo_info['id'],
install_group='dud', # Replace with a group that includes createrepo and mkisofs
setup_dns=True,
bind_opts={'dirs': {'/dev': '/dev', }})
broot.workdir = self.workdir
# create the mock chroot
self.logger.info("Initializing dud buildroot")
broot.init()
self.logger.info("dud buildroot ready: " + broot.rootdir())
# user repos
repos = self.opts.get('repos', [])
# buildroot repo
path_info = koji.PathInfo(topdir=self.options.topurl)
repopath = path_info.repo(repo_info['id'], target_info['build_tag_name'])
baseurl = '%s/%s' % (repopath, arch)
self.logger.info('BASEURL: %s' % baseurl)
repos.append(baseurl)
imgdata = {
'arch': arch,
'task_id': self.id,
'name': '',
'version': '',
'release': '',
'logs': [],
'rpmlist': [],
'files': [],
}
# Donwload each and every one of the packages on the list. We allow more than one
# rpms per DUD ISO. Do them one by one to report which one may fail
for rpm in pkg_list:
cmd = ['/usr/bin/dnf']
if self.opts.get('alldeps'):
cmd.extend([
'download', '--resolve', '--alldeps', rpm,
])
else:
cmd.extend([
'download', rpm,
])
rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
if rv:
raise koji.GenericError("DUD build failed while getting the involved rpm '{}': {}".format(rpm, str(rv)))
# Create the dd directory structure.
cmd = ['/usr/bin/mkdir']
cmd.extend([
'-p', './dd/rpms/x86_64/repodata/',
'-p', './dd/src/',
])
rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
if rv:
raise koji.GenericError("DUD build failed while preparing the dir struct for the ISO: " + str(rv))
# Inspiration from https://pagure.io/koji/blob/master/f/plugins/builder/runroot.py#_201 for this dirty hack
cmd = ['/usr/bin/sh', '-c']
cmd.extend([
'/usr/bin/echo -e "Driver Update Disk version 3\c" > ./dd/rhdd3',
])
rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
if rv:
raise koji.GenericError("DUD build failed while writing the rhdd3 file in the ISO: " + str(rv))
imgdata['name'] = dud_name
imgdata['version'] = dud_version
imgdata['release'] = dud_release
# Get the SCM content into the ISO root
# Retrieve SCM content if it exists
if self.opts.get('scmurl'):
# get configuration
scm = SCM(self.opts.get('scmurl'))
scm.assert_allowed(allowed=self.options.allowed_scms,
session=self.session,
by_config=self.options.allowed_scms_use_config,
by_policy=self.options.allowed_scms_use_policy,
policy_data={
'user_id': self.taskinfo['owner'],
'channel': self.session.getChannel(self.taskinfo['channel_id'],
strict=True)['name'],
'scratch': self.opts.get('scratch')
})
logfile = os.path.join(self.workdir, 'checkout-%s.log' % arch)
self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
build_tag=build_tag, scratch=self.opts.get('scratch'))
scmdir = broot.tmpdir()
koji.ensuredir(scmdir)
scmsrcdir = scm.checkout(scmdir, self.session,
self.getUploadDir(), logfile)
self.run_callbacks("postSCMCheckout",
scminfo=scm.get_info(),
build_tag=build_tag,
scratch=self.opts.get('scratch'),
srcdir=scmsrcdir)
cmd = ['/usr/bin/cp']
cmd.extend([
'-aR', os.path.basename(scmsrcdir), './dd/',
])
rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
if rv:
raise koji.GenericError("DUD build failed while copying SCM repo content into dir struct: " + str(rv))
# Get the RPMs inside the corresponding dir struct for the ISO
cmd = ['/usr/bin/sh', '-c']
# Could not get it to work with a more elegant syntax, as it would not find the *.rpm files otherwise
cmd.extend([
'/usr/bin/cp *.rpm ./dd/rpms/x86_64/',
])
rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
if rv:
raise koji.GenericError("DUD build failed while copying RPMs into dir struct: " + str(rv))
cmd = ['/usr/bin/createrepo']
cmd.extend([
'-q', '--workers=1', './dd/rpms/x86_64/',
])
rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
if rv:
raise koji.GenericError("DUD build failed while creating ISO repodata: " + str(rv))
# mkisofs -quiet -lR -V OEMDRV -input-charset utf8 -o $PACKNAME ./dd
cmd = ['/usr/bin/mkisofs']
iso_name = 'dd-{name}-{version}-{release}.{arch}.iso'.format(name=dud_name, version=dud_version, release=dud_release, arch=arch)
cmd.extend([
'-quiet', '-lR', '-V OEMDRV', '-input-charset', 'utf8', '-o', iso_name, './dd',
])
rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
if rv:
raise koji.GenericError("DUD build failed while mkisofs: " + str(rv))
fpath = os.path.join(broot.tmpdir(), iso_name)
img_file = os.path.basename(fpath)
self.uploadFile(fpath, remoteName=os.path.basename(img_file))
imgdata['files'].append(img_file)
if not self.opts.get('scratch'):
hdrlist = self.getImagePackagesFromPath(broot.tmpdir())
broot.markExternalRPMs(hdrlist)
imgdata['rpmlist'] = hdrlist
broot.expire()
self.logger.error("Uploading image data: %s", imgdata)
return imgdata

74
plugins/cli/dud.py Normal file
View file

@ -0,0 +1,74 @@
from optparse import OptionParser
from koji import canonArch
from koji.plugin import export_cli
from koji_cli.lib import (
_running_in_bg,
activate_session,
watch_tasks,
)
# All client related stuff, to be located in ~/.koji/plugins/dud.py
@export_cli
def handle_dud_build(goptions, session, args):
"[build] Run a command in a buildroot"
usage = "usage: %prog dud-build [options] <koji_target> --scmurl=<git_repo> <dud_name> <version> <pkg(nvr optional)> [<pkg> ...]"
usage += "\n(Specify the --help global option for a list of other help options)"
parser = OptionParser(usage=usage)
parser.add_option("--scratch", action="store_true", default=False,
help="Perform a scratch build")
parser.add_option("--scmurl", metavar="SCMURL", default=None,
help="SCM repository URL for non-rpm related content to be included in the ISO")
parser.add_option("--alldeps", action="store_true", default=False,
help="Download all involved rpm dependencies and put them inside the DUD ISO as well")
parser.add_option("--arch", action="append", dest="arches", default=[],
help="Limit arches to this subset")
parser.add_option("--can-fail", action="store", dest="optional_arches",
metavar="ARCH1,ARCH2,...", default="",
help="List of archs which are not blocking for build "
"(separated by commas.")
parser.add_option("--nowait", action="store_false", dest="wait", default=True)
parser.add_option("--wait", action="store_true",
help="Wait on the image creation, even if running in the background")
(options, args) = parser.parse_args(args)
if len(args) < 4:
parser.error("Incorrect number of arguments")
assert False # pragma: no cover
pkg_list = []
# Can't use * operator with unpacking in Python 2.7, but this works for both Python 2 and 3
target, dud_name, dud_version, pkg_list = args[0], args[1], args[2], args[3:]
activate_session(session, goptions)
kwargs = {
'scratch': options.scratch,
'alldeps': options.alldeps,
'scmurl': options.scmurl,
'optional_arches': [canonArch(arch)
for arch in options.optional_arches.split(',')
if arch],
}
arches = []
if options.arches:
arches = [canonArch(arch) for arch in options.arches]
task_id = session.dudBuild(
target=target,
arches=arches,
dud_version=dud_version,
dud_name=dud_name,
pkg_list=pkg_list,
**kwargs)
if not goptions.quiet:
print("Created task: %d" % task_id)
print("Task info: %s/taskinfo?taskID=%s" % (goptions.weburl, task_id))
if options.wait or (options.wait is None and not _running_in_bg()):
session.logout()
return watch_tasks(session, [task_id], quiet=goptions.quiet,
poll_interval=goptions.poll_interval, topurl=goptions.topurl)

40
plugins/hub/dud.py Normal file
View file

@ -0,0 +1,40 @@
import koji
import koji.tasks
import kojihub
from koji.context import context
from koji.plugin import export
koji.tasks.LEGACY_SIGNATURES['dudBuild'] = [
[['dud_name', 'dud_version', 'arches', 'target', 'pkg_list', 'opts'],
None, None, (None,)]]
koji.tasks.LEGACY_SIGNATURES['createDudIso'] = [
[['dud_name', 'dud_version', 'dud_release', 'arch',
'target_info', 'build_tag', 'repo_info', 'pkg_list', 'opts'],
None, None, (None,)]]
# /usr/lib/koji-hub-plugins/
@export
def dudBuild(dud_name, dud_version, arches, target, pkg_list, optional_arches=None, scratch=False, alldeps=False, scmurl=None, priority=None):
context.session.assertPerm('image')
taskOpts = {
'channel': 'image',
}
if priority:
if priority < 0:
if not context.session.hasPerm('admin'):
raise koji.ActionNotAllowed(
'only admins may create high-priority tasks')
taskOpts['priority'] = koji.PRIO_DEFAULT + priority
opts = {
'scratch': scratch,
'alldeps': alldeps,
'scmurl': scmurl,
'optional_arches': optional_arches,
}
return kojihub.make_task('dudBuild',
[dud_name, dud_version, arches, target, pkg_list, opts],
**taskOpts)