basic kiwi support

plugins/builder/kiwi.py

@@ -0,0 +1,387 @@
+import glob
+import pickle
+import os
+import xml.dom.minidom
+from fnmatch import fnmatch
+
+import koji
+from koji.util import joinpath, to_list
+from koji.tasks import ServerExit
+from __main__ import BaseBuildTask, BuildImageTask, BuildRoot, SCM
+
+
+class KiwiBuildTask(BuildImageTask):
+    Methods = ['kiwiBuild']
+    _taskWeight = 4.0
+
+    def get_nvr(self, desc_path):
+        # TODO: update release in desc
+        kiwi_files = glob.glob('%s/*.kiwi' % desc_path)
+        if len(kiwi_files) != 1:
+            raise koji.GenericError("Repo must contain only one .kiwi file.")
+
+        cfg = kiwi_files[0]
+
+        newxml = xml.dom.minidom.parse(cfg)
+        image = newxml.getElementsByTagName('image')[0]
+
+        name = image.getAttribute('name')
+        version = None
+        release = None
+        for preferences in image.getElementsByTagName('preferences'):
+            try:
+                version = preferences.getElementsByTagName('version')[0].childNodes[0].data
+            except Exception:
+                pass
+            try:
+                release = preferences.getElementsByTagName('release')[0].childNodes[0].data
+            except Exception:
+                release = None
+        if not version:
+            raise koji.BuildError("Description file doesn't contain preferences/version")
+        return name, version, release
+
+    def handler(self, target, arches, desc_url, desc_path, opts=None):
+        target_info = self.session.getBuildTarget(target, strict=True)
+        build_tag = target_info['build_tag']
+        repo_info = self.getRepo(build_tag)
+        # check requested arches against build tag
+        buildconfig = self.session.getBuildConfig(build_tag)
+        if not buildconfig['arches']:
+            raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
+        tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
+        if arches:
+            for arch in arches:
+                if koji.canonArch(arch) not in tag_archlist:
+                    raise koji.BuildError("Invalid arch for build tag: %s" % arch)
+        else:
+            arches = tag_archlist
+
+        if not opts:
+            opts = {}
+        if not opts.get('scratch'):
+            opts['scratch'] = False
+        if not opts.get('optional_arches'):
+            opts['optional_arches'] = []
+        self.opts = opts
+
+        # get configuration
+        scm = SCM(desc_url)
+        scm.assert_allowed(allowed=self.options.allowed_scms,
+                           session=self.session,
+                           by_config=self.options.allowed_scms_use_config,
+                           by_policy=self.options.allowed_scms_use_policy,
+                           policy_data={
+                               'user_id': self.taskinfo['owner'],
+                               'channel': self.session.getChannel(self.taskinfo['channel_id'],
+                                                                  strict=True)['name'],
+                               'scratch': opts['scratch'],
+                           })
+        logfile = os.path.join(self.workdir, 'checkout.log')
+        self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
+                           build_tag=build_tag, scratch=opts['scratch'])
+        scmdir = self.workdir
+        koji.ensuredir(scmdir)
+        scmsrcdir = scm.checkout(scmdir, self.session,
+                                 self.getUploadDir(), logfile)
+        self.run_callbacks("postSCMCheckout",
+                           scminfo=scm.get_info(),
+                           build_tag=build_tag,
+                           scratch=opts['scratch'],
+                           srcdir=scmsrcdir)
+
+        path = os.path.join(scmsrcdir, desc_path)
+        name, version, release = self.get_nvr(path)
+
+        bld_info = {}
+        if not opts['scratch']:
+            bld_info = self.initImageBuild(name, version, release, target_info, opts)
+            release = bld_info['release']
+        elif not release:
+            release = self.session.getNextRelease({'name': name, 'version': version})
+
+        try:
+            subtasks = {}
+            canfail = []
+            self.logger.debug("Spawning jobs for image arches: %r" % (arches))
+            for arch in arches:
+                subtasks[arch] = self.session.host.subtask(
+                    method='createKiwiImage',
+                    arglist=[name, version, release, arch,
+                             target_info, build_tag, repo_info,
+                             desc_url, desc_path, opts],
+                    label=arch, parent=self.id, arch=arch)
+                if arch in self.opts['optional_arches']:
+                    canfail.append(subtasks[arch])
+            self.logger.debug("Got image subtasks: %r" % (subtasks))
+            self.logger.debug("Waiting on image subtasks (%s can fail)..." % canfail)
+            results = self.wait(to_list(subtasks.values()), all=True,
+                                failany=True, canfail=canfail)
+
+            # if everything failed, fail even if all subtasks are in canfail
+            self.logger.debug('subtask results: %r', results)
+            all_failed = True
+            for result in results.values():
+                if not isinstance(result, dict) or 'faultCode' not in result:
+                    all_failed = False
+                    break
+            if all_failed:
+                raise koji.GenericError("all subtasks failed")
+
+            # determine ignored arch failures
+            ignored_arches = set()
+            for arch in arches:
+                if arch in self.opts['optional_arches']:
+                    task_id = subtasks[arch]
+                    result = results[task_id]
+                    if isinstance(result, dict) and 'faultCode' in result:
+                        ignored_arches.add(arch)
+
+            self.logger.debug('Image Results for hub: %s' % results)
+            results = dict([(str(k), v) for k, v in results.items()])
+            if opts['scratch']:
+                self.session.host.moveImageBuildToScratch(self.id, results)
+            else:
+                self.session.host.completeImageBuild(self.id, bld_info['id'], results)
+        except (SystemExit, ServerExit, KeyboardInterrupt):
+            # we do not trap these
+            raise
+        except Exception:
+            if not opts['scratch']:
+                if bld_info:
+                    self.session.host.failBuild(self.id, bld_info['id'])
+            raise
+
+        # tag it
+        if not opts['scratch'] and not opts.get('skip_tag'):
+            tag_task_id = self.session.host.subtask(method='tagBuild',
+                                                    arglist=[target_info['dest_tag'],
+                                                             bld_info['id'], False, None, True],
+                                                    label='tag', parent=self.id, arch='noarch')
+            self.wait(tag_task_id)
+
+        # report results
+        report = ''
+        if opts['scratch']:
+            respath = ', '.join(
+                [os.path.join(koji.pathinfo.work(),
+                              koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()])
+            report += 'Scratch '
+        else:
+            respath = koji.pathinfo.imagebuild(bld_info)
+        report += 'image build results in: %s' % respath
+        return report
+
+
+class KiwiCreateImageTask(BaseBuildTask):
+    Methods = ['createKiwiImage']
+    _taskWeight = 2.0
+
+    def prepareDescription(self, desc_path, release, repos):
+        # TODO: update release in desc
+        kiwi_files = glob.glob('%s/*.kiwi' % desc_path)
+        if len(kiwi_files) != 1:
+            raise koji.GenericError("Repo must contain only one .kiwi file.")
+
+        cfg = kiwi_files[0]
+
+        newxml = xml.dom.minidom.parse(cfg)
+        image = newxml.getElementsByTagName('image')[0]
+
+        # remove old repos
+        for old_repo in image.getElementsByTagName('repository'):
+            image.removeChild(old_repo)
+
+        # add new ones
+        for repo in sorted(set(repos)):
+            repo_node = newxml.createElement('repository')
+            repo_node.setAttribute('type', 'rpm-md')
+            source = newxml.createElement('source')
+            source.setAttribute('path', repo)
+            repo_node.appendChild(source)
+            image.appendChild(repo_node)
+
+        # TODO: release is part of version (major.minor.release)
+        # preferences = image.getElementsByTagName('preferences')[0]
+        # try:
+        #    preferences.getElementsByTagName('release')[0].childNodes[0].data = release
+        # except Exception:
+        #    rel_node = newxml.createElement('release')
+        #    rel_node.data = release
+        #    preferences.appendChild(rel_node)
+
+        types = []
+        for pref in image.getElementsByTagName('preferences'):
+            for type in pref.getElementsByTagName('type'):
+                # TODO: if type.getAttribute('primary') == 'true':
+                types.append(type.getAttribute('image'))
+
+        # write file back
+        with open(cfg, 'wt') as f:
+            f.write(newxml.toprettyxml())
+ 
+        return cfg, types
+
+    def getImagePackagesFromCache(self, cachepath):
+        """
+        Read RPM header information from the yum cache available in the
+        given path. Returns a list of dictionaries for each RPM included.
+        """
+        found = False
+        hdrlist = {}
+        fields = ['name', 'version', 'release', 'epoch', 'arch',
+                  'buildtime', 'sigmd5']
+        for root, dirs, files in os.walk(cachepath):
+            for f in files:
+                if fnmatch(f, '*.rpm'):
+                    pkgfile = os.path.join(root, f)
+                    hdr = koji.get_header_fields(pkgfile, fields)
+                    hdr['size'] = os.path.getsize(pkgfile)
+                    hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
+                    del hdr['sigmd5']
+                    hdrlist[os.path.basename(pkgfile)] = hdr
+                    found = True
+        if not found:
+            raise koji.LiveCDError('No repos found in yum cache!')
+        return list(hdrlist.values())
+
+    def getImagePackages(self, result):
+        """Proper handler for getting rpminfo from result list,
+        it need result list to contain payloadhash, etc. to work correctly"""
+        hdrlist = []
+        for line in open(result, 'rt'):
+            line = line.strip()
+            name, epoch, version, release, arch, disturl, license = line.split('|')
+            try:
+                # "(none)" for None epochs
+                epoch = int(epoch)
+            except ValueError:
+                epoch = None
+            hdrlist.append({
+                'name': name,
+                'epoch': epoch,
+                'version': version,
+                'release': release,
+                'arch': arch,
+                'payloadhash': '',
+                'size': 0,
+                'buildtime': 0,
+            })
+
+        return hdrlist
+
+    def handler(self, name, version, release, arch,
+                target_info, build_tag, repo_info,
+                desc_url, desc_path, opts=None):
+        self.opts = opts
+        build_tag = target_info['build_tag']
+        broot = BuildRoot(self.session, self.options,
+                          tag=build_tag,
+                          arch=arch,
+                          task_id=self.id,
+                          repo_id=repo_info['id'],
+                          install_group='kiwi',
+                          setup_dns=True,
+                          bind_opts={'dirs': {'/dev': '/dev', }})
+        broot.workdir = self.workdir
+
+        # create the mock chroot
+        self.logger.debug("Initializing kiwi buildroot")
+        broot.init()
+        self.logger.debug("Kiwi buildroot ready: " + broot.rootdir())
+
+        # get configuration
+        scm = SCM(desc_url)
+        scm.assert_allowed(allowed=self.options.allowed_scms,
+                           session=self.session,
+                           by_config=self.options.allowed_scms_use_config,
+                           by_policy=self.options.allowed_scms_use_policy,
+                           policy_data={
+                               'user_id': self.taskinfo['owner'],
+                               'channel': self.session.getChannel(self.taskinfo['channel_id'],
+                                                                  strict=True)['name'],
+                               'scratch': self.opts.get('scratch')
+                           })
+        logfile = os.path.join(self.workdir, 'checkout-%s.log' % arch)
+        self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
+                           build_tag=build_tag, scratch=self.opts.get('scratch'))
+        scmdir = broot.tmpdir()
+        koji.ensuredir(scmdir)
+        scmsrcdir = scm.checkout(scmdir, self.session,
+                                 self.getUploadDir(), logfile)
+        self.run_callbacks("postSCMCheckout",
+                           scminfo=scm.get_info(),
+                           build_tag=build_tag,
+                           scratch=self.opts.get('scratch'),
+                           srcdir=scmsrcdir)
+
+        # user repos
+        repos = self.opts.get('repos', [])
+        # buildroot repo
+        path_info = koji.PathInfo(topdir=self.options.topurl)
+        repopath = path_info.repo(repo_info['id'], target_info['build_tag_name'])
+        baseurl = '%s/%s' % (repopath, arch)
+        self.logger.debug('BASEURL: %s' % baseurl)
+        repos.append(baseurl)
+
+        path = os.path.join(scmsrcdir, desc_path)
+        desc, types = self.prepareDescription(path, release, repos)
+        self.uploadFile(desc)
+
+        cmd = ['kiwi-ng']
+        if self.opts.get('profile'):
+            cmd.extend(['--profile', self.opts['profile']])
+        target_dir = '/builddir/result/image'
+        cmd.extend([
+            'system', 'build',
+            '--description', os.path.join(os.path.basename(scmsrcdir), desc_path),
+            '--target-dir', target_dir,
+        ])
+        rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
+        if rv:
+            raise koji.GenericError("Kiwi failed")
+
+        result = pickle.load(open(joinpath(broot.rootdir(), target_dir[1:], 'kiwi.result'), 'rb'))
+
+        imgdata = {
+            'arch': arch,
+            'task_id': self.id,
+            'logs': [
+                os.path.basename(desc)
+            ],
+            'name': name,
+            'version': version,
+            'release': release,
+            'rpmlist': [],
+            'files': [],
+        }
+        # TODO: upload detailed log?
+        # build/image-root.log
+        # os.path.join(broot.tmpdir(), target_dir[1:], "build/image-root.log")
+
+        # for type in types:
+        #     img_file = '%s.%s-%s.%s' % (name, version, arch, type)
+        #     self.uploadFile(os.path.join(broot.rootdir()), remoteName=img_file)
+        #     imgdata['files'].append(img_file)
+        fpath = os.path.join(broot.rootdir(),
+                             result.result_files['disk_format_image'].filename[1:])
+        img_file = os.path.basename(fpath)
+        self.uploadFile(fpath, remoteName=os.path.basename(img_file))
+        imgdata['files'].append(img_file)
+
+        if not self.opts.get('scratch'):
+            if False:
+                # should be used after kiwi update
+                fpath = os.path.join(broot.rootdir(),
+                                     result.result_files['image_packages'].filename[1:])
+                hdrlist = self.getImagePackages(fpath)
+            else:
+                cachepath = os.path.join(broot.rootdir(), 'var/cache/kiwi/dnf')
+                hdrlist = self.getImagePackagesFromCache(cachepath)
+            broot.markExternalRPMs(hdrlist)
+            imgdata['rpmlist'] = hdrlist
+
+        broot.expire()
+
+        self.logger.error("Uploading image data: %s", imgdata)
+        return imgdata

plugins/cli/kiwi.py

@@ -0,0 +1,75 @@
+import os
+from optparse import OptionParser
+
+from koji import canonArch
+
+from koji.plugin import export_cli
+from koji_cli.lib import (
+    _,
+    _running_in_bg,
+    activate_session,
+    watch_tasks,
+)
+
+
+@export_cli
+def handle_kiwi_build(goptions, session, args):
+    "[build] Run a command in a buildroot"
+    usage = _("usage: %prog kiwi-build [options] "
+              "<target> <description_scm> <description_path>")
+    usage += _("\n(Specify the --help global option for a list of other help options)")
+    parser = OptionParser(usage=usage)
+    parser.add_option("--scratch", action="store_true", default=False,
+                      help=_("Perform a scratch build"))
+    parser.add_option("--repo", action="append",
+                      help=_("Specify a repo that will override the repo used to install "
+                             "RPMs in the image. May be used multiple times. The "
+                             "build tag repo associated with the target is the default."))
+    parser.add_option("--noprogress", action="store_true",
+                      help=_("Do not display progress of the upload"))
+    parser.add_option("--kiwi-profile", action="store", default=None,
+                      help=_("Select profile from description file"))
+    parser.add_option("--can-fail", action="store", dest="optional_arches",
+                      metavar="ARCH1,ARCH2,...", default="",
+                      help=_("List of archs which are not blocking for build "
+                             "(separated by commas."))
+    parser.add_option("--arch", action="append", dest="arches", default=[],
+                      help=_("Limit arches to this subset"))
+    parser.add_option("--nowait", action="store_false", dest="wait", default=True)
+    parser.add_option("--wait", action="store_true",
+                      help=_("Wait on the image creation, even if running in the background"))
+    (options, args) = parser.parse_args(args)
+
+    if len(args) != 3:
+        parser.error(_("Incorrect number of arguments"))
+        assert False  # pragma: no cover
+    target, scm, path = args
+
+    activate_session(session, goptions)
+
+    kwargs = {
+        'scratch': options.scratch,
+        'optional_arches': [canonArch(arch)
+                            for arch in options.optional_arches.split(',')
+                            if arch],
+        'profile': options.kiwi_profile,
+    }
+
+    arches = []
+    if options.arches:
+        arches = [canonArch(arch) for arch in options.arches]
+
+    task_id = session.kiwiBuild(
+        target=target,
+        arches=arches,
+        desc_url=scm,
+        desc_path=path,
+        **kwargs)
+
+    if not goptions.quiet:
+        print("Created task: %d" % task_id)
+        print("Task info: %s/taskinfo?taskID=%s" % (goptions.weburl, task_id))
+    if options.wait or (options.wait is None and not _running_in_bg()):
+        session.logout()
+        return watch_tasks(session, [task_id], quiet=goptions.quiet,
+                           poll_interval=goptions.poll_interval, topurl=goptions.topurl)

plugins/hub/kiwi.py

@@ -0,0 +1,38 @@
+import koji
+import koji.tasks
+import kojihub
+
+from koji.context import context
+from koji.plugin import export
+
+koji.tasks.LEGACY_SIGNATURES['kiwiBuild'] = [
+    [['target', 'arches', 'desc_url', 'desc_path', 'opts'],
+     None, None, (None,)]]
+koji.tasks.LEGACY_SIGNATURES['createKiwiImage'] = [
+    [['name', 'version', 'release', 'arch',
+      'target_info', 'build_tag', 'repo_info', 'desc_url', 'opts'],
+     None, None, (None,)]]
+
+
+@export
+def kiwiBuild(target, arches, desc_url, desc_path, optional_arches=None, profile=None,
+              scratch=False, priority=None):
+    context.session.assertPerm('image')
+    taskOpts = {
+        'channel': 'image',
+    }
+    if priority:
+        if priority < 0:
+            if not context.session.hasPerm('admin'):
+                raise koji.ActionNotAllowed(
+                    'only admins may create high-priority tasks')
+        taskOpts['priority'] = koji.PRIO_DEFAULT + priority
+
+    opts = {
+        'optional_arches': optional_arches,
+        'profile': profile,
+        'scratch': scratch,
+    }
+    return kojihub.make_task('kiwiBuild',
+                             [target, arches, desc_url, desc_path, opts],
+                             **taskOpts)