#!/usr/bin/python2

# Koji build daemon
# Copyright (c) 2005-2014 Red Hat, Inc.
#
#    Koji is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation;
#    version 2.1 of the License.
#
#    This software is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this software; if not, write to the Free Software
#    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
#
# Authors:
#       Mike McLean <mikem@redhat.com>
#       Mike Bonnet <mikeb@redhat.com>

from __future__ import absolute_import, division

import copy
import errno
import filecmp
import glob
import grp
import io
import json
import logging
import logging.handlers
import os
import pwd
import random
import re
import shutil
import signal
import smtplib
import socket
import subprocess
import sys
import time
import traceback
import xml.dom.minidom
import zipfile
from fnmatch import fnmatch
from gzip import GzipFile
from optparse import SUPPRESS_HELP, OptionParser

try:
    # Due to https://bugzilla.redhat.com/show_bug.cgi?id=1923971
    # and https://pagure.io/koji/issue/2964
    # guestfs needs to be imported before dnf, so json libraries will
    # load in non-breaking order (guestfs would be otherwise imported
    # by ImageFactory/Oz
    import guestfs  # noqa: F401
except ImportError:
    pass
import dnf

import Cheetah.Template
import librepo
import requests
import rpm
import six
import six.moves.xmlrpc_client
from multilib import multilib

import koji
import koji.arch
import koji.plugin
import koji.rpmdiff
import koji.tasks
import koji.util
from koji.daemon import SCM, TaskManager, incremental_upload, log_output
from koji.tasks import (
    BaseTaskHandler,
    MultiPlatformTask,
    ServerExit,
    ServerRestart,
    RefuseTask,
)
from koji.util import (
    dslice,
    dslice_ex,
    format_shell_cmd,
    isSuccess,
    joinpath,
    parseStatus,
    to_list,
)

try:
    import requests_gssapi as reqgssapi
    Krb5Error = reqgssapi.exceptions.RequestException
except ImportError:  # pragma: no cover
    try:
        import requests_kerberos as reqgssapi
        Krb5Error = reqgssapi.exceptions.RequestException
    except ImportError:  # pragma: no cover
        reqgssapi = None

# imports for LiveCD, LiveMedia, and Appliance handler
try:
    import pykickstart.parser as ksparser
    import pykickstart.handlers.control as kscontrol
    import pykickstart.errors as kserrors
    import iso9660  # from pycdio
    image_enabled = True
except ImportError:  # pragma: no cover
    image_enabled = False

try:
    from imgfac.BuildDispatcher import BuildDispatcher
    from imgfac.Builder import Builder
    from imgfac.PluginManager import PluginManager
    from imgfac.ReservationManager import ReservationManager
    plugin_mgr = PluginManager('/etc/imagefactory/plugins.d')
    plugin_mgr.load()
    from imgfac.ApplicationConfiguration import ApplicationConfiguration
    from imgfac.PersistentImageManager import PersistentImageManager
    from imgfac.BaseImage import BaseImage
    from imgfac.TargetImage import TargetImage
    # NOTE: import below requires Factory 1.1.7 or higher
    from imgfac.FactoryUtils import qemu_convert_cmd
    ozif_enabled = True
except ImportError:  # pragma: no cover
    ozif_enabled = False


def main(options, session):
    logger = logging.getLogger("koji.build")
    logger.info('Starting up')
    koji.util.setup_rlimits(options.__dict__, logger)
    tm = TaskManager(options, session)
    tm.findHandlers(globals())
    tm.findHandlers(vars(koji.tasks))
    if options.plugin:
        # load plugins
        pt = koji.plugin.PluginTracker(path=options.pluginpath.split(':'))
        for name in options.plugin:
            logger.info('Loading plugin: %s' % name)
            tm.scanPlugin(pt.load(name))

    if options.single:
        # force all task handlers to run in Foreground
        for key in tm.handlers:
            tm.handlers[key].Foreground = True

    def shutdown(*args):
        raise SystemExit

    def restart(*args):
        logger.warning("Initiating graceful restart")
        tm.restart_pending = True
    signal.signal(signal.SIGTERM, shutdown)
    signal.signal(signal.SIGUSR1, restart)
    exit_code = 0
    taken = False
    while True:
        try:
            if not taken:
                tm.updateBuildroots()
            tm.updateTasks()
            taken = tm.getNextTask()
        except (SystemExit, ServerExit, KeyboardInterrupt):
            logger.warning("Exiting")
            break
        except ServerRestart:
            logger.warning("Restarting")
            os.execv(sys.argv[0], sys.argv)
        except koji.AuthExpired:
            logger.error('Session expired')
            exit_code = 1
            break
        except koji.RetryError:
            raise
        except koji.AuthError:
            logger.error('Authentication error')
            exit_code = 1
            break
        except Exception:
            # XXX - this is a little extreme
            # log the exception and continue
            logger.error(''.join(traceback.format_exception(*sys.exc_info())))
            taken = False
        try:
            if not taken:
                # Only sleep if we didn't take a task, otherwise retry immediately.
                # The load-balancing code in getNextTask() will prevent a single builder
                # from getting overloaded.
                logger.debug('Sleeping for %s', options.sleeptime)
                time.sleep(options.sleeptime)
            elif options.single:
                # exit after first task
                break
        except (SystemExit, KeyboardInterrupt):
            logger.warning("Exiting")
            break
    logger.warning("Shutting down, please wait...")
    tm.shutdown()
    session.logout()
    sys.exit(exit_code)


class BuildRoot(object):

    def __init__(self, session, options, *args, **kwargs):
        self.logger = logging.getLogger("koji.build.buildroot")
        self.session = session
        self.options = options
        self.logs = set()
        if len(args) + len(kwargs) == 1:
            # manage an existing mock buildroot
            self._load(*args, **kwargs)
        else:
            self._new(*args, **kwargs)

    def _load(self, data):
        # manage an existing buildroot
        if isinstance(data, dict):
            # assume data already pulled from db
            self.id = data['id']
        else:
            self.id = data
            data = self.session.getBuildroot(self.id)
        self.task_id = data['task_id']
        self.tag_id = data['tag_id']
        self.tag_name = data['tag_name']
        self.repoid = data['repo_id']
        self.repo_info = self.session.repoInfo(self.repoid, strict=True)
        self.event_id = self.repo_info['create_event']
        self.br_arch = data['arch']
        self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self)
        self.config = self.session.getBuildConfig(self.tag_id, event=self.event_id)

    def _new(self, tag, arch, task_id, repo_id=None, install_group='build',
             setup_dns=False, bind_opts=None, maven_opts=None, maven_envs=None,
             deps=None, internal_dev_setup=None):
        """Create a brand new repo"""
        if not repo_id:
            raise koji.BuildrootError("A repo id must be provided")
        repo_info = self.session.repoInfo(repo_id, strict=True)
        self.repo_info = repo_info
        self.repoid = self.repo_info['id']
        self.event_id = self.repo_info['create_event']
        self.task_id = task_id
        self.config = self.session.getBuildConfig(tag, event=self.event_id)
        if not self.config:
            raise koji.BuildrootError("Could not get config info for tag: %s" % tag)
        self.tag_id = self.config['id']
        self.tag_name = self.config['name']
        if self.config['id'] != repo_info['tag_id']:
            raise koji.BuildrootError("tag/repo mismatch: %s vs %s"
                                      % (self.config['name'], repo_info['tag_name']))
        repo_state = koji.REPO_STATES[repo_info['state']]
        if repo_state == 'EXPIRED':
            # This should be ok. Expired repos are still intact, just not
            # up-to-date (which may be the point in some cases).
            self.logger.info("Requested repo (%i) is no longer current" % repo_id)
        elif repo_state != 'READY':
            raise koji.BuildrootError("Requested repo (%i) is %s" % (repo_id, repo_state))
        self.br_arch = koji.canonArch(arch)
        # armhfp is not a valid arch according to autoconf
        if arch == 'armhfp':
            self.target_arch = 'armv7hl'
        else:
            self.target_arch = arch
        self.logger.debug("New buildroot: %(tag_name)s/%(br_arch)s/%(repoid)s" % vars(self))
        id = self.session.host.newBuildRoot(self.repoid, self.br_arch, task_id=task_id)
        if id is None:
            raise koji.BuildrootError("failed to get a buildroot id")
        self.id = id
        self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self)
        self.install_group = install_group
        self.setup_dns = setup_dns
        self.bind_opts = bind_opts
        self.maven_opts = maven_opts
        self.maven_envs = maven_envs
        self.deps = deps
        self.internal_dev_setup = internal_dev_setup
        self._writeMockConfig()

    def _writeMockConfig(self):
        # mock config
        configdir = '/etc/mock/koji'
        configfile = "%s/%s.cfg" % (configdir, self.name)
        self.mockcfg = "koji/%s" % self.name

        opts = {}
        for k in ('repoid', 'tag_name'):
            if hasattr(self, k):
                opts[k] = getattr(self, k)
        for k in ('mockdir', 'topdir', 'topurl', 'topurls', 'packager', 'vendor',
                  'distribution', 'mockhost', 'yum_proxy', 'rpmbuild_timeout'):
            if hasattr(self.options, k):
                opts[k] = getattr(self.options, k)
        opts['buildroot_id'] = self.id
        if self.setup_dns:
            opts['rpmbuild_networking'] = True
        opts['use_host_resolv'] = self.setup_dns
        opts['install_group'] = self.install_group
        opts['maven_opts'] = self.maven_opts
        opts['maven_envs'] = self.maven_envs
        opts['bind_opts'] = self.bind_opts
        opts['target_arch'] = self.target_arch
        if 'mock.forcearch' in self.config['extra']:
            if bool(self.config['extra']['mock.forcearch']):
                opts['forcearch'] = self.target_arch
        if 'mock.package_manager' in self.config['extra']:
            opts['package_manager'] = self.config['extra']['mock.package_manager']
        if 'mock.yum.module_hotfixes' in self.config['extra']:
            opts['module_hotfixes'] = self.config['extra']['mock.yum.module_hotfixes']
        if 'mock.yum.best' in self.config['extra']:
            opts['yum_best'] = int(self.config['extra']['mock.yum.best'])
        # Append opts['plugin_conf'] to enable Mock package signing
        if 'mock.plugin_conf.sign_enable' in self.config['extra']:
            # check rest of configuration
            if ('mock.plugin_conf.sign_opts.cmd' not in self.config['extra'] or
               'mock.plugin_conf.sign_opts.opts' not in self.config['extra']):
                raise koji.GenericError("Tag is not configured properly for mock's sign plugin'")
            opts['plugin_conf'] = {
                'sign_enable': self.config['extra']['mock.plugin_conf.sign_enable'],
                'sign_opts': {
                    'cmd': self.config['extra']['mock.plugin_conf.sign_opts.cmd'],
                    'opts': self.config['extra']['mock.plugin_conf.sign_opts.opts'],
                }
            }
        if 'mock.module_setup_commands' in self.config['extra']:
            opts['module_setup_commands'] = self.config['extra']['mock.module_setup_commands']
        if 'mock.releasever' in self.config['extra']:
            opts['releasever'] = self.config['extra']['mock.releasever']
        if self.internal_dev_setup is not None:
            opts['internal_dev_setup'] = bool(self.internal_dev_setup)
        opts['tag_macros'] = {}
        opts['tag_envvars'] = {}
        for key in self.config['extra']:
            if key.startswith('rpm.macro.'):
                macro = '%' + key[10:]
                opts['tag_macros'][macro] = str(self.config['extra'][key])
            elif key.startswith('rpm.env.'):
                opts['tag_envvars'][key[8:]] = str(self.config['extra'][key])
        if 'mock.use_bootstrap' in self.config['extra']:
            opts['use_bootstrap'] = bool(self.config['extra']['mock.use_bootstrap'])

        # it must be allowed in kojid.conf *and* in tag's extra info
        if (not self.options.mock_bootstrap_image and
           self.config['extra'].get('mock.bootstrap_image')):
            self.logger.warning("Mock bootstrap image requested by buildroot %d, "
                                "but forbidden on builder" % self.id)
        opts['bootstrap_image'] = self.options.mock_bootstrap_image and \
            self.config['extra'].get('mock.bootstrap_image')

        output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts)

        # write config
        with koji._open_text_file(configfile, 'wt') as fo:
            fo.write(output)

        self.single_log(configfile, name='mock_config.log')

    def get_repo_dir(self):
        pathinfo = koji.PathInfo(topdir='')
        return pathinfo.repo(self.repoid, self.tag_name)

    def _repositoryEntries(self, pi, plugin=False):
        entries = []
        if plugin:
            tag_name = 'pluginRepository'
        else:
            tag_name = 'repository'
        id_suffix = 'repo'
        name_prefix = 'Repository for Koji'
        for dep in self.deps:
            if isinstance(dep, six.integer_types):
                # dep is a task ID, the url points to the task output directory
                repo_type = 'task'
                dep_url = pi.task(dep)
                snapshots = 'true'
            else:
                # dep is a build NVR, the url points to the build output directory
                repo_type = 'build'
                build = koji.parse_NVR(dep)
                dep_url = pi.mavenbuild(build)
                snapshots = 'false'
            repo_id = 'koji-%(repo_type)s-%(dep)s-%(id_suffix)s' % locals()
            entry = """
        <%(tag_name)s>
          <id>%(repo_id)s</id>
          <name>%(name_prefix)s %(repo_type)s %(dep)s</name>
          <url>%(dep_url)s</url>
          <layout>default</layout>
          <releases>
            <enabled>true</enabled>
            <updatePolicy>never</updatePolicy>
            <checksumPolicy>fail</checksumPolicy>
          </releases>
          <snapshots>
            <enabled>%(snapshots)s</enabled>
            <updatePolicy>never</updatePolicy>
            <checksumPolicy>fail</checksumPolicy>
          </snapshots>
        </%(tag_name)s>""" % locals()
            entries.append((repo_id, entry))
        return entries

    def writeMavenSettings(self, destfile, outputdir):
        """
        Write the Maven settings.xml file to the specified destination.
        """
        task_id = self.task_id
        repo_id = self.repoid
        tag_name = self.tag_name
        deploy_dir = outputdir[len(self.rootdir()):]

        pi = koji.PathInfo(topdir=self.options.topurl)
        repourl = pi.repo(repo_id, tag_name) + '/maven'

        mirror_spec = '*'
        settings = """<settings xmlns="http://maven.apache.org/POM/4.0.0"
  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
                      http://maven.apache.org/xsd/settings-1.0.0.xsd">
  <interactiveMode>false</interactiveMode>
  <mirrors>
    <mirror>
      <id>koji-maven-repo-%(tag_name)s-%(repo_id)i</id>
      <name>Koji-managed Maven repository (%(tag_name)s-%(repo_id)i)</name>
      <url>%(repourl)s</url>
      <mirrorOf>%(mirror_spec)s</mirrorOf>
    </mirror>
  </mirrors>
  <profiles>
    <profile>
      <id>koji-task-%(task_id)s</id>
      <properties>
        <altDeploymentRepository>koji-output::default::file://%(deploy_dir)s</altDeploymentRepository>
      </properties>"""
        if self.deps:
            settings += """
      <repositories>"""
            for dep_repo_id, dep_repo_entry in self._repositoryEntries(pi):
                mirror_spec += ',!' + dep_repo_id
                settings += dep_repo_entry
            settings += """
      </repositories>
      <pluginRepositories>"""
            for dep_repo_id, dep_repo_entry in self._repositoryEntries(pi, plugin=True):
                mirror_spec += ',!' + dep_repo_id
                settings += dep_repo_entry
            settings += """
      </pluginRepositories>"""
        settings += """
    </profile>
  </profiles>
  <activeProfiles>
    <activeProfile>koji-task-%(task_id)s</activeProfile>
  </activeProfiles>
</settings>
"""
        settings = settings % locals()
        with koji._open_text_file(self.rootdir() + destfile, 'wt') as fo:
            fo.write(settings)

    def mock(self, args):
        """Run mock"""
        mockpath = getattr(self.options, "mockpath", "/usr/bin/mock")
        cmd = [mockpath, "-r", self.mockcfg]
        # if self.options.debug_mock:
        #    cmd.append('--debug')
        # TODO: should we pass something like --verbose --trace instead?
        if 'mock.new_chroot' in self.config['extra']:
            if self.config['extra']['mock.new_chroot']:
                cmd.append('--new-chroot')
            else:
                cmd.append('--old-chroot')
        if self.config['extra'].get('mock.bootstrap_image_ready', False):
            cmd.append('--bootstrap-image-ready')
        cmd.extend(args)
        self.logger.info(format_shell_cmd(cmd))
        workdir = getattr(self, 'workdir', None)
        mocklog = 'mock_output.log'
        pid = os.fork()
        if pid:
            log_patterns = [
                '%s/*.log' % self.resultdir(),
                '%s/var/log/dnf*.log' % self.rootdir(),
            ]
            if workdir:
                log_patterns.append('%s/%s' % (workdir, mocklog))
            logs = BuildRootLogs(self, log_patterns, with_ts=self.options.log_timestamps)

            finished = False
            while not finished:
                time.sleep(1)
                status = os.waitpid(pid, os.WNOHANG)
                if status[0] != 0:
                    finished = True
                logs.sync_logs()

            # clean up and return exit status of command
            logs.close_logs()
            return status[1]

        else:
            # in no case should exceptions propagate past here
            try:
                self.session._forget()
                if workdir:
                    outfile = os.path.join(workdir, mocklog)
                    flags = os.O_CREAT | os.O_WRONLY | os.O_APPEND
                    fd = os.open(outfile, flags, 0o666)
                    os.dup2(fd, 1)
                    os.dup2(fd, 2)
                if os.getuid() == 0 and hasattr(self.options, "mockuser"):
                    self.logger.info('Running mock as %s' % self.options.mockuser)
                    uid, gid = pwd.getpwnam(self.options.mockuser)[2:4]
                    os.setgroups([grp.getgrnam('mock')[2]])
                    os.setregid(gid, gid)
                    os.setreuid(uid, uid)
                os.execvp(cmd[0], cmd)
            except BaseException:
                # diediedie
                print("Failed to exec mock")
                print(''.join(traceback.format_exception(*sys.exc_info())))
                os._exit(1)

    def getUploadPath(self):
        """Get the path that should be used when uploading files to
        the hub."""
        return koji.pathinfo.taskrelpath(self.task_id)

    def incremental_log(self, fname, fd):
        ret = incremental_upload(self.session, fname, fd, self.getUploadPath(), logger=self.logger)
        self.logs.add(fname)
        return ret

    def single_log(self, localfile, name=None):
        if name is None:
            name = os.path.basename(localfile)
        self.session.uploadWrapper(localfile, self.getUploadPath(), name=name)
        self.logs.add(name)

    def init(self):
        rv = self.mock(['--init'])

        if rv:
            self.expire()
            raise koji.BuildrootError("could not init mock buildroot, %s" % self._mockResult(rv))
        # log kernel version
        self.mock(['--chroot', 'uname -r'])
        self.session.host.setBuildRootList(self.id, self.getPackageList())

    def _mockResult(self, rv, logfile=None):
        if logfile:
            pass
        elif os.WIFEXITED(rv) and os.WEXITSTATUS(rv) == 10:
            logfile = 'build.log'
        elif os.WIFEXITED(rv) and os.WEXITSTATUS(rv) == 1:
            logfile = 'build.log or root.log'
        else:
            logfile = 'root.log'
        msg = '; see %s for more information' % logfile
        return parseStatus(rv, 'mock') + msg

    def rebuild_srpm(self, srpm):
        self.session.host.setBuildRootState(self.id, 'BUILDING')

        # unpack SRPM to tempdir
        srpm_dir = os.path.join(self.tmpdir(), 'srpm_unpacked')
        koji.ensuredir(srpm_dir)
        top_dir = self.path_without_to_within(srpm_dir)
        args = ['--no-clean', '--target', 'noarch', '--chroot', '--',
                'rpm', '--define', '_topdir %s' % top_dir, '-iv', srpm]
        rv = self.mock(args)

        # find specfile
        spec_files = glob.glob("%s/SPECS/*.spec" % srpm_dir)
        if len(spec_files) == 0:
            raise koji.BuildError("No spec file found")
        elif len(spec_files) > 1:
            raise koji.BuildError("Multiple spec files found: %s" % spec_files)
        spec_file = os.path.join(top_dir, "SPECS", os.path.basename(spec_files[0]))

        # rebuild SRPM from spec + sources
        args = ['--no-clean', '--target', 'noarch', '--chroot', '--',
                'rpmbuild', '--define', '_topdir %s' % top_dir, '-bs', '--nodeps', spec_file]
        rv = self.mock(args)

        result_dir = os.path.join(srpm_dir, 'SRPMS')
        for fn in glob.glob('%s/*.src.rpm' % result_dir):
            shutil.move(os.path.join(result_dir, fn), self.resultdir())

        if rv:
            self.expire()
            raise koji.BuildError("error building srpm, %s" % self._mockResult(rv))

    def build_srpm(self, specfile, sourcedir, source_cmd):
        self.session.host.setBuildRootState(self.id, 'BUILDING')
        if source_cmd:
            # call the command defined by source_cmd in the chroot so any required files not stored
            # in the SCM can be retrieved
            chroot_sourcedir = sourcedir[len(self.rootdir()):]
            args = ['--no-clean', '--unpriv', '--cwd', chroot_sourcedir, '--chroot']
            args.extend(source_cmd)
            rv = self.mock(args)
            if rv:
                self.expire()
                raise koji.BuildError("error retrieving sources, %s" % self._mockResult(rv))

        alt_sources_dir = "%s/SOURCES" % sourcedir
        if self.options.support_rpm_source_layout and os.path.isdir(alt_sources_dir):
            sources_dir = alt_sources_dir
        else:
            sources_dir = sourcedir

        args = ['--no-clean', '--buildsrpm', '--spec', specfile, '--sources', sources_dir,
                '--target', 'noarch']

        rv = self.mock(args)

        if rv:
            self.expire()
            raise koji.BuildError("error building srpm, %s" % self._mockResult(rv))

    def build(self, srpm, arch=None):
        # run build
        self.session.host.setBuildRootState(self.id, 'BUILDING')
        args = ['--no-clean']
        if arch:
            args.extend(['--target', arch])
        args.extend(['--rebuild', srpm])
        rv = self.mock(args)

        self.session.host.updateBuildRootList(self.id, self.getPackageList())
        if rv:
            self.expire()
            raise koji.BuildError("error building package (arch %s), %s" %
                                  (arch, self._mockResult(rv)))

    def getPackageList(self):
        """Return a list of packages from the buildroot

        Each member of the list is a dictionary containing the following fields:
            - id, optional for internal rpm available in rpmlist.jsonl
            - name
            - version
            - release
            - epoch
            - arch
            - payloadhash
            - size
            - buildtime
            - external_repo, optional for external rpm
            - location, optional for external rpm
        """
        fields = ('name',
                  'version',
                  'release',
                  'epoch',
                  'arch',
                  'sigmd5',
                  'size',
                  'buildtime')
        # Determine db path
        dbpath = "%s/usr/lib/sysimage/rpm" % self.rootdir()
        if not os.path.exists(dbpath):
            # older variant (<rhel10, <fedora39)
            dbpath = "%s/var/lib/rpm" % self.rootdir()
        else:
            # if new dbpath is used and mock is running older rpm on builder
            # _db* symlinks could exist in migrated directory
            for f in glob.glob('%s/__db*' % dbpath):
                if not os.path.isfile(f):
                    os.unlink(f)
        if not os.path.exists(dbpath):
            raise koji.GenericError("Can't get list of installed rpms")
        rpm.addMacro("_dbpath", dbpath)
        ret = []
        try:
            ts = rpm.TransactionSet()
            for h in ts.dbMatch():
                pkg = koji.get_header_fields(h, fields)
                # skip our fake packages
                if pkg['name'] in ['buildsys-build', 'gpg-pubkey']:
                    # XXX config
                    continue
                pkg['payloadhash'] = koji.hex_string(pkg['sigmd5'])
                del pkg['sigmd5']
                ret.append(pkg)
        finally:
            rpm.delMacro("_dbpath")
        self.markExternalRPMs(ret)
        self.mapInternalRPMs(ret)
        return ret

    def getMavenPackageList(self, repodir):
        """Return a list of Maven packages that were installed into the local repo
        to satisfy build requirements.

        Each member of the list is a dictionary containing the following fields:
        - maven_info: a dict of Maven info containing the groupId, artifactId, and version fields
        - files: a list of files associated with that POM
        """
        packages = []
        for path, dirs, files in os.walk(repodir):
            relpath = path[len(repodir) + 1:]
            maven_files = []
            for repofile in files:
                if koji.util.multi_fnmatch(repofile, self.options.maven_repo_ignore) or \
                        koji.util.multi_fnmatch(os.path.join(relpath, repofile),
                                                self.options.maven_repo_ignore):
                    continue
                if relpath == '' and repofile in ['scm-sources.zip', 'patches.zip']:
                    # special-case the archives of the sources and patches, since we drop them in
                    # root of the output directory
                    continue
                maven_files.append({'path': relpath, 'filename': repofile,
                                    'size': os.path.getsize(os.path.join(path, repofile))})
            if maven_files:
                path_comps = relpath.split('/')
                if len(path_comps) < 3:
                    raise koji.BuildrootError('files found in unexpected path in local Maven repo,'
                                              ' directory: %s, files: %s' %
                                              (relpath,
                                               ', '.join([f['filename'] for f in maven_files])))
                # extract the Maven info from the path within the local repo
                maven_info = {'version': path_comps[-1],
                              'artifact_id': path_comps[-2],
                              'group_id': '.'.join(path_comps[:-2])}
                packages.append({'maven_info': maven_info, 'files': maven_files})

        return packages

    def mavenBuild(self, sourcedir, outputdir, repodir,
                   props=None, profiles=None, options=None, goals=None):
        self.session.host.setBuildRootState(self.id, 'BUILDING')
        cmd = ['--no-clean', '--chroot', '--unpriv', '--cwd', sourcedir[len(self.rootdir()):],
               '--', '/usr/bin/mvn', '-C']
        if options:
            cmd.extend(options)
        if profiles:
            cmd.append('-P%s' % ','.join(profiles))
        if props:
            for name, value in props.items():
                if value is not None:
                    cmd.append('-D%s=%s' % (name, value))
                else:
                    cmd.append('-D%s' % name)
        if goals:
            cmd.extend(goals)
        cmd.extend(['deploy'])
        rv = self.mock(cmd)

        # if the deploy command failed, don't raise an error on unknown artifacts, because that
        # will mask the underlying failure
        ignore_unknown = False
        if rv:
            ignore_unknown = True
        self.session.host.updateMavenBuildRootList(self.id, self.task_id,
                                                   self.getMavenPackageList(repodir),
                                                   ignore=self.getMavenPackageList(outputdir),
                                                   project=True, ignore_unknown=ignore_unknown,
                                                   extra_deps=self.deps)
        if rv:
            self.expire()
            raise koji.BuildrootError('error building Maven package, %s' %
                                      self._mockResult(rv, logfile='root.log'))

    def markExternalRPMs(self, rpmlist):
        """Check rpms against pkgorigins and add external repo data to the external ones

        Modifies rpmlist in place. No return
        """
        external_repos = self.session.getExternalRepoList(self.repo_info['tag_id'],
                                                          event=self.repo_info['create_event'])
        if not external_repos:
            # nothing to do
            return
        # index external repos by expanded url
        erepo_idx = {}
        for erepo in external_repos:
            # substitute $arch in the url with the arch of the repo we're generating
            ext_url = erepo['url'].replace('$arch', self.br_arch)
            erepo_idx[ext_url] = erepo

        opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')])
        opts['tempdir'] = self.options.workdir

        repo_url = os.path.join(self.get_repo_dir(), self.br_arch)
        # repo_url can start with '/', don't use os.path.join
        if self.options.topurl:
            repo_url = '%s/%s' % (self.options.topurl, repo_url)
        elif self.options.topdir:
            repo_url = '%s/%s' % (self.options.topdir, repo_url)
        self.logger.info("repo url of buildroot: %s is %s", self.name, repo_url)

        tmpdir = os.path.join(self.tmpdir(), 'librepo-markExternalRPMs')
        koji.ensuredir(tmpdir)
        h = librepo.Handle()
        r = librepo.Result()
        h.setopt(librepo.LRO_REPOTYPE, librepo.LR_YUMREPO)
        h.setopt(librepo.LRO_URLS, [repo_url])
        h.setopt(librepo.LRO_DESTDIR, tmpdir)
        # We are using this just to find out location of 'origin',
        # we don't even need to download it since we use openRemoteFile
        h.setopt(librepo.LRO_YUMDLIST, [])
        h.perform(r)
        pkgorigins = r.getinfo(librepo.LRR_YUM_REPOMD)['origin']['location_href']
        koji.util.rmtree(tmpdir)

        relpath = os.path.join(self.get_repo_dir(), self.br_arch, pkgorigins)
        with koji.openRemoteFile(relpath, **opts) as fo:
            # at this point we know there were external repos at the create event,
            # so there should be an origins file.
            origin_idx = {}
            with GzipFile(fileobj=fo, mode='r') as fo2:
                if six.PY3:
                    fo2 = io.TextIOWrapper(fo2, encoding='utf-8')
                for line in fo2:
                    parts = line.split(None, 2)
                    if len(parts) < 2:
                        continue
                    # first field is formated by yum as [e:]n-v-r.a
                    nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" % koji.parse_NVRA(parts[0])
                    origin_idx[nvra] = parts[1]
        # mergerepo starts from a local repo in the task workdir, so internal
        # rpms have an odd-looking origin that we need to look for
        localtail = '/repo_%s_premerge/' % self.repo_info['id']
        for rpm_info in rpmlist:
            key = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info
            # src rpms should not show up in rpmlist so we do not have to
            # worry about fixing the arch for them
            ext_url = origin_idx.get(key)
            if not ext_url:
                raise koji.BuildError("No origin for %s" % key)
            erepo = erepo_idx.get(ext_url)
            if not erepo:
                if ext_url.startswith('file://') and ext_url.endswith(localtail):
                    # internal rpm
                    continue
                raise koji.BuildError("Unknown origin for %s: %s" % (key, ext_url))
            rpm_info['external_repo'] = erepo
            rpm_info['location'] = erepo['external_repo_id']

    def mapInternalRPMs(self, rpmlist):
        """
        Map each rpm item of rpmlist to a specific koji rpm entry based on repo contents

        The rpmList should be a list of dicts containing rpm header values. These entries will be
        modified in place to include an id field when mapped.

        This mapping relies on the rpmlist.jsonl file for the repo. If this file is missing, the
        code will fall back to querying the hub.

        This function will raise an error if there is a sigmd5 mismatch for a given rpm.

        :param list rpmlist: rpm list fetched from local RPMDB.
        :return: None
        """

        opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')])
        rpmlist_path = os.path.join(self.get_repo_dir(), self.br_arch, 'rpmlist.jsonl')
        compat_mode = False
        try:
            with koji.openRemoteFile(rpmlist_path, **opts) as fo:
                repo_rpms = [json.loads(line) for line in fo]
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 404:
                self.logger.warning("Missing repo content file: %s", rpmlist_path)
                # TODO: remove this workaround once we can assume that repos contain this file
                repo_rpms = self.repo_draft_rpms()
                compat_mode = True
            else:
                raise
        fmt = "%(name)s-%(version)s-%(release)s.%(arch)s"
        repo_rpms = {fmt % r: r for r in repo_rpms}
        for rpm_info in rpmlist:
            if 'external_repo' in rpm_info:
                continue
            nvra = fmt % rpm_info
            data = repo_rpms.get(nvra)
            if not data:
                # happens a lot in compat mode because we only query for drafts
                if not compat_mode:
                    self.logger.warning("%s not found in rpmlist.jsonl", nvra)
                continue
            # check payloadhash in case they are different
            elif data['payloadhash'] != rpm_info['payloadhash']:
                raise koji.BuildrootError(
                    "RPM: %s: payloadhash: %s mismatch expected %s in rpmlist.jsonl"
                    % (nvra, rpm_info['payloadhash'], data['payloadhash'])
                )
            else:
                # set rpm id
                rpm_info['id'] = data['id']

    def repo_draft_rpms(self):
        drafts, draftbuilds = self.session.listTaggedRPMS(
            tag=self.repo_info['tag_id'],
            event=self.repo_info['create_event'],
            latest=True,
            draft=True)
        return drafts

    def path_without_to_within(self, path):
        """
        Convert an absolute path from without the BuildRoot to one within.

        For example, if the BuildRoot is located at '/tmp/my/build/root',
        calling path_without_to_within('/tmp/my/build/root/foo/bar') would
        return '/foo/bar').

        :param path:
            A reference within the BuildRoot but as an absolute path from
            without a chroot.
        :return:
            The equivalent absolute path from within a chroot of the BuildRoot.
        """
        root = self.rootdir()
        if os.path.commonprefix([root, path]) != root:
            raise ValueError(
                'path %r is not within the BuildRoot at %r' % (path, root)
            )
        return os.path.join('/', os.path.relpath(path, root))

    def resultdir(self):
        return "%s/%s/result" % (self.options.mockdir, self.name)

    def rootdir(self):
        return "%s/%s/root" % (self.options.mockdir, self.name)

    def tmpdir(self, within=False):
        # mock 1.4+ /tmp is tmpfs mounted on each run, different
        # directory is needed for persistency
        # 'within' is equivalent to broot.path_without_to_within(broot.tmpdir())
        base = self.options.chroot_tmpdir
        if within:
            return base
        else:
            return "%s%s" % (self.rootdir(), base)

    def expire(self):
        self.session.host.setBuildRootState(self.id, 'EXPIRED')


class BuildRootLogs(object):

    "Track the logs generated during a mock run"

    def __init__(self, broot, patterns, with_ts=False):
        self.broot = broot
        self.patterns = patterns
        self.with_ts = with_ts

        self.loginfo = {}
        self.ts_logs = {}
        self.ignored = set()
        self.names = {}
        self.logger = broot.logger
        self.workdir = getattr(broot, 'workdir', None)
        if with_ts and self.workdir is None:
            self.logger.error('No workdir defined -- disabling log timestamps')
            self.with_ts = False

    def find_logs(self):
        matches = []
        for pattern in self.patterns:
            m = glob.glob(pattern)
            for path in m:
                if path not in self.loginfo:
                    self.logger.debug('Log matched pattern %r: %s', pattern, path)
                    matches.append(path)
        return matches

    def add_log(self, path):
        if path in self.loginfo or path in self.ignored:
            return
        if path.endswith('-ts.log'):
            self.logger.error('ignoring stray ts log: %s', path)
            self.ignored.add(path)
            return

        # pick a unique name for upload if there is overlap
        fname = os.path.basename(path)
        if fname in self.names:
            base, ext = os.path.splitext(fname)
            for n in range(99):
                fname = '%s.DUP%02i%s' % (base, n, ext)
                if fname not in self.names:
                    self.logger.debug('Using log name alias %s for %s', fname, path)
                    break
            else:
                self.logger.error('Unable to find unique log name for %s', path)
                self.ignored.add(path)
                return

        info = {'name': fname, 'path': path}
        self.names[fname] = info
        self.loginfo[path] = info
        self.logger.debug('Watching buildroot log: %r', info)

        if self.with_ts:
            self.add_ts_log(info)

    def add_ts_log(self, info):
        ts_name = '%(name)s-ts.log' % info
        ts_path = os.path.join(self.workdir, ts_name)

        offset = 0
        if os.path.exists(ts_path):
            # XXX should this even happen?
            # read last offset from existing ts file
            with koji._open_text_file(ts_path) as ts_file:
                lines = ts_file.readlines()
                if lines:
                    offset = int(lines[-1].split()[1])
        else:
            # initialize ts file at zero
            with koji._open_text_file(ts_path, 'at') as ts_file:
                ts_file.write('%.0f 0\n' % time.time())

        info['offset'] = offset
        info['ts_log'] = ts_path
        self.ts_logs[ts_path] = {'name': ts_name, 'path': ts_path, 'ts': True}
        self.logger.debug('Watching timestamp log: %r', info)

    def get_logs(self):
        for info in self.loginfo.values():
            yield info
        for info in self.ts_logs.values():
            yield info

    def sync_logs(self):
        paths = self.find_logs()

        for fpath in paths:
            self.add_log(fpath)

        for info in self.get_logs():
            # note that the ts logs are listed last
            try:
                self.sync_log(info)
            except OSError:
                self.logger.error("Error reading mock log: %(path)s", info)
                self.logger.error(''.join(traceback.format_exception(*sys.exc_info())))
                continue

    def sync_log(self, info):
        fpath = info['path']
        try:
            st = os.stat(fpath)
        except OSError as e:
            if e.errno == errno.ENOENT:
                if info.get('missing'):
                    # we've already noted this, don't spam the logs
                    return
                self.logger.error('Log disappeared: %(path)s', info)
                info['missing'] = True
                return
            raise
        if info.get('missing'):
            self.logger.error('Log re-appeared: %(path)s', info)
            del info['missing']
        fd = info.get('fd')
        if fd is None:
            # freshly added, we need to open it
            fd = open(fpath, 'rb')
            info['fd'] = fd
        last_st = info.get('st')
        if last_st:
            if st.st_ino != last_st.st_ino or st.st_size < last_st.st_size:
                # file appears to have been rewritten or truncated
                self.logger.info('Rereading %s, inode: %s -> %s, size: %s -> %s',
                                 fpath, last_st.st_ino, st.st_ino, last_st.st_size, st.st_size)
                fd.close()
                fd = open(fpath, 'rb')
                info['fd'] = fd
        info['st'] = st

        self.broot.incremental_log(info['name'], fd)

        ts_log = info.get('ts_log')
        if ts_log and self.with_ts:
            # race condition against incremental_upload's tell,
            # but with enough precision for ts.log purposes
            position = fd.tell()
            info.setdefault('offset', 0)
            if info['offset'] < position:
                with koji._open_text_file(ts_log, 'at') as ts_log:
                    ts_log.write('%.0f %i\n' % (time.time(), position))
                info['offset'] = position

    def close_logs(self):
        for info in self.get_logs():
            fd = info.get('fd')
            if fd:
                fd.close()


class ChainBuildTask(BaseTaskHandler):

    Methods = ['chainbuild']
    # mostly just waiting on other tasks
    _taskWeight = 0.1

    def handler(self, srcs, target, opts=None):
        """Run a chain build

            target and opts are passed on to the build tasks
            srcs is a list of "build levels"
            each build level is a list of strings, each string may be one of:
                - a build src (SCM url only)
                - an n-v-r
            each build level is processed in order
            successive levels are only started once the previous levels have completed
            and gotten into the repo.
        """
        if opts.get('scratch'):
            raise koji.BuildError("--scratch is not allowed with chain-builds")
        target_info = self.session.getBuildTarget(target)
        if not target_info:
            raise koji.GenericError('unknown build target: %s' % target)
        nvrs = []
        for n_level, build_level in enumerate(srcs):
            # if there are any nvrs to wait on, do so
            if nvrs:
                task_id = self.session.host.subtask(method='waitrepo',
                                                    arglist=[
                                                        target_info['build_tag_name'], None, nvrs],
                                                    label="wait %i" % n_level,
                                                    parent=self.id)
                self.wait(task_id, all=True, failany=True)
            nvrs = []
            # kick off the builds for this level
            build_tasks = []
            for n_src, src in enumerate(build_level):
                if SCM.is_scm_url(src):
                    task_id = self.session.host.subtask(method='build',
                                                        arglist=[src, target, opts],
                                                        label="build %i,%i" % (n_level, n_src),
                                                        parent=self.id)
                    build_tasks.append(task_id)
                else:
                    nvrs.append(src)
                    # next pass will wait for these
            if build_tasks:
                # the level could have been all nvrs
                self.wait(build_tasks, all=True, failany=True)
            # see what builds we created in this batch so the next pass can wait for them also
            for build_task in build_tasks:
                builds = self.session.listBuilds(taskID=build_task)
                if builds:
                    nvrs.append(builds[0]['nvr'])


class BuildTask(BaseTaskHandler):

    Methods = ['build']
    # we mostly just wait on other tasks
    _taskWeight = 0.2

    def handler(self, src, target, opts=None):
        """Handler for the master build task"""
        if opts is None:
            opts = {}
        self.opts = opts
        if opts.get('arch_override') and not opts.get('scratch'):
            raise koji.BuildError("arch_override is only allowed for scratch builds")
        if opts.get('repo_id') is not None:
            repo_info = self.session.repoInfo(opts['repo_id'])
            if not repo_info:
                raise koji.BuildError('No such repo: %s' % opts['repo_id'])
            repo_state = koji.REPO_STATES[repo_info['state']]
            if repo_state not in ('READY', 'EXPIRED'):
                raise koji.BuildError('Bad repo: %s (%s)' % (repo_info['id'], repo_state))
            self.event_id = repo_info['create_event']
        else:
            repo_info = None
            # we'll wait for a repo later (self.getRepo)
            self.event_id = None

        if opts.get('custom_user_metadata'):
            if not isinstance(opts['custom_user_metadata'], dict):
                raise koji.BuildError('custom_user_metadata must be serializable to a JSON object')

            try:
                json.dumps(opts['custom_user_metadata'])
            except TypeError:
                error_msg = 'custom_user_metadata is not JSON serializable'
                self.logger.exception(error_msg)
                raise koji.BuildError(error_msg)

        task_info = self.session.getTaskInfo(self.id)
        target_info = None
        if target:
            target_info = self.session.getBuildTarget(target, event=self.event_id)
        if target_info:
            dest_tag = target_info['dest_tag']
            build_tag = target_info['build_tag']
            if repo_info is not None:
                # make sure specified repo matches target
                if repo_info['tag_id'] != target_info['build_tag']:
                    raise koji.BuildError('Repo/Target mismatch: %s/%s'
                                          % (repo_info['tag_name'], target_info['build_tag_name']))
        else:
            # if repo_id is specified, we can allow the 'target' arg to simply specify
            # the destination tag (since the repo specifies the build tag).
            if repo_info is None:
                raise koji.GenericError('unknown build target: %s' % target)
            build_tag = repo_info['tag_id']
            if target is None:
                # ok, call it skip-tag for the buildroot tag
                self.opts['skip_tag'] = True
                dest_tag = build_tag
            else:
                taginfo = self.session.getTag(target, event=self.event_id)
                if not taginfo:
                    raise koji.GenericError('neither tag nor target: %s' % target)
                dest_tag = taginfo['id']
        # policy checks...
        policy_data = {
            'user_id': task_info['owner'],
            'source': src,
            'task_id': self.id,
            'build_tag': build_tag,  # id
            'skip_tag': bool(self.opts.get('skip_tag')),
            'scratch': opts.get('scratch'),
            'draft': opts.get('draft'),
            'from_scm': SCM.is_scm_url(src),
            'repo_id': opts.get('repo_id'),
        }
        if target_info:
            policy_data['target'] = target_info['name']
        if not self.opts.get('skip_tag'):
            policy_data['tag'] = dest_tag  # id
        # backward-compatible deprecated policies (TODO: remove if py2 is dropped - rhel6 builders)
        if not SCM.is_scm_url(src) and not opts.get('scratch'):
            # let hub policy decide
            self.session.host.assertPolicy('build_from_srpm', policy_data)
        if opts.get('repo_id') is not None:
            # use of this option is governed by policy
            self.session.host.assertPolicy('build_from_repo_id', policy_data)
        self.session.host.assertPolicy('build_rpm', policy_data)
        if not repo_info:
            repo_info = self.getRepo(build_tag, builds=opts.get('wait_builds'),
                                     wait=opts.get('wait_repo'))  # (subtask)
            self.event_id = self.session.getLastEvent()['id']
        srpm = self.getSRPM(src, build_tag, repo_info['id'])
        h = self.readSRPMHeader(srpm)
        data = koji.get_header_fields(h, ['name', 'version', 'release', 'epoch'])
        data['task_id'] = self.id
        if getattr(self, 'source', False):
            data['source'] = self.source['source']
            data['extra'] = {'source': {'original_url': self.source['url']}}

        if opts.get('custom_user_metadata'):
            data.setdefault('extra', {})
            data['extra']['custom_user_metadata'] = opts['custom_user_metadata']

        extra_arches = None
        self.logger.info("Reading package config for %(name)s" % data)
        pkg_cfg = self.session.getPackageConfig(dest_tag, data['name'], event=self.event_id)
        self.logger.debug("%r" % pkg_cfg)
        if pkg_cfg is not None:
            extra_arches = pkg_cfg.get('extra_arches')
        if not self.opts.get('skip_tag') and not self.opts.get('scratch'):
            # Make sure package is on the list for this tag
            if pkg_cfg is None:
                raise koji.BuildError("package %s not in list for tag %s"
                                      % (data['name'], target_info['dest_tag_name']))
            elif pkg_cfg['blocked']:
                raise koji.BuildError("package %s is blocked for tag %s"
                                      % (data['name'], target_info['dest_tag_name']))
            # TODO - more pre tests
        archlist = self.getArchList(build_tag, h, extra=extra_arches)
        # pass draft option in
        if opts.get('draft'):
            data['draft'] = opts.get('draft')
        # let the system know about the build we're attempting
        if not self.opts.get('scratch'):
            # scratch builds do not get imported
            build_id = self.session.host.initBuild(data)
        # (initBuild raises an exception if there is a conflict)
        failany = (self.opts.get('fail_fast', False) or
                   not getattr(self.options, 'build_arch_can_fail', False))
        try:
            self.extra_information = {"src": src, "data": data, "target": target}
            srpm, rpms, brmap, logs = self.runBuilds(srpm, build_tag, archlist,
                                                     repo_info['id'], failany=failany)

            if opts.get('scratch'):
                # scratch builds do not get imported
                self.session.host.moveBuildToScratch(self.id, srpm, rpms, logs=logs)
            else:
                self.session.host.completeBuild(self.id, build_id, srpm, rpms, brmap, logs=logs)
        except (SystemExit, ServerExit, KeyboardInterrupt):
            # we do not trap these
            raise
        except Exception:
            if not self.opts.get('scratch'):
                # scratch builds do not get imported
                self.session.host.failBuild(self.id, build_id)
            # reraise the exception
            raise
        if not self.opts.get('skip_tag') and not self.opts.get('scratch'):
            self.tagBuild(build_id, dest_tag)

    def getSRPM(self, src, build_tag, repo_id):
        """
        Get srpm from src - it can fetch SCM and build SRPM from there or
        alternatively get downloaded SRPM (and rebuild it if needed).

        Buildroot has extra.rebuild_srpm field with default value of True. For
        scratch builds we allow overriding by user, for regular builds only
        buildtag's option can affect it.

        :param str src:
           SCM url or filename
        :param str|dict build_tag:
           build tag used for re/building srpm
        :param int repo_id:
           repo id to be used
        """
        if isinstance(src, str):
            if SCM.is_scm_url(src):
                return self.getSRPMFromSCM(src, build_tag, repo_id)
            else:
                buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
                rebuild = buildconfig['extra'].get('rebuild_srpm', True)
                if self.opts.get('scratch') and self.opts.get('rebuild_srpm') is not None:
                    rebuild = self.opts.get('rebuild_srpm')
                if rebuild:
                    return self.getSRPMFromSRPM(src, build_tag, repo_id)
                else:
                    return src
        else:
            raise koji.BuildError('Invalid source specification: %s' % src)
            # XXX - other methods?

    def getSRPMFromSRPM(self, src, build_tag, repo_id):
        # rebuild srpm in mock, so it gets correct disttag, rpm version, etc.
        taskarch = self.choose_taskarch('noarch', None, build_tag)
        task_id = self.session.host.subtask(method='rebuildSRPM',
                                            arglist=[src, build_tag, {
                                                'repo_id': repo_id,
                                                'scratch': self.opts.get('scratch')}],
                                            label='srpm',
                                            arch=taskarch,
                                            parent=self.id)
        # wait for subtask to finish
        result = self.wait(task_id)[task_id]
        if 'source' in result:
            self.source = result['source']
        else:
            self.logger.warning('subtask did not provide source data')
        srpm = result['srpm']
        return srpm

    def getSRPMFromSCM(self, url, build_tag, repo_id):
        # TODO - allow different ways to get the srpm
        taskarch = self.choose_taskarch('noarch', None, build_tag)
        task_id = self.session.host.subtask(method='buildSRPMFromSCM',
                                            arglist=[url, build_tag, {
                                                'repo_id': repo_id,
                                                'scratch': self.opts.get('scratch')}],
                                            label='srpm',
                                            arch=taskarch,
                                            parent=self.id)
        # wait for subtask to finish
        result = self.wait(task_id)[task_id]
        if 'source' in result:
            self.source = result['source']
        else:
            self.logger.warning('subtask did not provide source data')
        srpm = result['srpm']
        return srpm

    def readSRPMHeader(self, srpm):
        # srpm arg should be a path relative to <BASEDIR>/work
        self.logger.debug("Reading SRPM")
        relpath = "work/%s" % srpm
        opts = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')])
        opts['tempdir'] = self.workdir
        with koji.openRemoteFile(relpath, **opts) as fo:
            h = koji.get_rpm_header(fo)
        if not koji.get_header_field(h, 'sourcepackage'):
            raise koji.BuildError("%s is not a source package" % srpm)
        return h

    def getArchList(self, build_tag, h, extra=None):
        # get list of arches to build for
        buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
        arches = buildconfig['arches']
        if not arches:
            # XXX - need to handle this better
            raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
        tag_archlist = [koji.canonArch(a) for a in arches.split()]
        self.logger.debug('arches: %s' % arches)
        if extra:
            self.logger.debug('Got extra arches: %s' % extra)
            arches = "%s %s" % (arches, extra)
        archlist = arches.split()
        self.logger.debug('base archlist: %r' % archlist)
        # - adjust arch list based on srpm macros
        buildarchs = koji.get_header_field(h, 'buildarchs')
        exclusivearch = koji.get_header_field(h, 'exclusivearch')
        excludearch = koji.get_header_field(h, 'excludearch')
        if buildarchs:
            archlist = buildarchs
            self.logger.debug('archlist after buildarchs: %r' % archlist)
        if exclusivearch:
            archlist = [a for a in archlist if a in exclusivearch]
            self.logger.debug('archlist after exclusivearch: %r' % archlist)
        if excludearch:
            archlist = [a for a in archlist if a not in excludearch]
            self.logger.debug('archlist after excludearch: %r' % archlist)
        # noarch is funny
        if 'noarch' not in excludearch and \
                ('noarch' in buildarchs or 'noarch' in exclusivearch):
            archlist.append('noarch')
        override = self.opts.get('arch_override')
        if self.opts.get('scratch') and override:
            # only honor override for scratch builds
            self.logger.debug('arch override: %s' % override)
            archlist = override.split()
        archdict = {}
        for a in archlist:
            # Filter based on canonical arches for tag
            # This prevents building for an arch that we can't handle
            if a == 'noarch' or koji.canonArch(a) in tag_archlist:
                archdict[a] = 1
        if not archdict:
            raise koji.BuildError("No matching arches were found")
        return to_list(archdict.keys())

    def choose_taskarch(self, arch, srpm, build_tag):
        """Adjust the arch for buildArch subtask as needed"""
        if koji.util.multi_fnmatch(arch, self.options.literal_task_arches):
            return arch
        if arch != 'noarch':
            return koji.canonArch(arch)

        # For noarch, attempt to honor ExcludeArch/ExclusiveArch
        # see https://pagure.io/koji/issue/19

        if srpm is None:
            exclusivearch = []
            excludearch = []
        else:
            h = self.readSRPMHeader(srpm)
            exclusivearch = koji.get_header_field(h, 'exclusivearch')
            excludearch = koji.get_header_field(h, 'excludearch')

        buildconfig = self.session.getBuildConfig(build_tag, event=self.event_id)
        noarch_arches = buildconfig.get('extra', {}).get('noarch_arches')

        if exclusivearch or excludearch or noarch_arches:
            # if one of the tag arches is filtered out, then we can't use a
            # noarch task
            arches = buildconfig['arches']
            tag_arches = [koji.canonArch(a) for a in arches.split()]
            exclusivearch = [koji.canonArch(a) for a in exclusivearch]
            excludearch = [koji.canonArch(a) for a in excludearch]
            # tag.extra overrides tag arches for noarch
            if noarch_arches:
                archlist = [koji.canonArch(a) for a in noarch_arches.split()]
                archlist = [a for a in archlist if a in tag_arches]
            else:
                archlist = list(tag_arches)
            if exclusivearch:
                archlist = [a for a in archlist if a in exclusivearch]
            if excludearch:
                archlist = [a for a in archlist if a not in excludearch]
            self.logger.info('Filtering arches for noarch subtask. Choices: %r', archlist)
            if not archlist:
                raise koji.BuildError("No valid arches were found. tag %r, extra %r,"
                                      "exclusive %r, exclude %r" % (tag_arches, noarch_arches,
                                                                    exclusivearch, excludearch))
            self.logger.debug('tag: %r, extra: %r, exclusive: %r, exclude: %r',
                              tag_arches, noarch_arches, exclusivearch, excludearch)
            if set(archlist) != set(tag_arches):
                return random.choice(archlist)
            else:
                # noarch is ok
                return 'noarch'

        # otherwise, noarch is ok
        return 'noarch'

    def runBuilds(self, srpm, build_tag, archlist, repo_id, failany=True):
        self.logger.debug("Spawning jobs for arches: %r" % (archlist))
        subtasks = {}
        keep_srpm = True
        for arch in archlist:
            taskarch = self.choose_taskarch(arch, srpm, build_tag)
            subtasks[arch] = self.session.host.subtask(method='buildArch',
                                                       arglist=[srpm, build_tag, arch,
                                                                keep_srpm, {'repo_id': repo_id}],
                                                       label=arch,
                                                       parent=self.id,
                                                       arch=taskarch)
            keep_srpm = False

        self.logger.debug("Got subtasks: %r" % (subtasks))
        self.logger.debug("Waiting on subtasks...")

        # wait for subtasks to finish
        results = self.wait(to_list(subtasks.values()), all=True, failany=failany)

        # finalize import
        # merge data into needed args for completeBuild call
        rpms = []
        brmap = {}
        logs = {}
        built_srpm = None
        for (arch, task_id) in six.iteritems(subtasks):
            result = results[task_id]
            self.logger.debug("DEBUG: %r : %r " % (arch, result,))
            brootid = result['brootid']
            for fn in result['rpms']:
                rpms.append(fn)
                brmap[fn] = brootid
            for fn in result['logs']:
                logs.setdefault(arch, []).append(fn)
            if result['srpms']:
                if built_srpm:
                    raise koji.BuildError("multiple builds returned a srpm.  task %i" % self.id)
                else:
                    built_srpm = result['srpms'][0]
                    brmap[result['srpms'][0]] = brootid
        if built_srpm:
            srpm = built_srpm
        else:
            raise koji.BuildError("could not find a built srpm")

        return srpm, rpms, brmap, logs

    def tagBuild(self, build_id, dest_tag):
        # XXX - need options to skip tagging and to force tagging
        # create the tagBuild subtask
        # this will handle the "post tests"
        task_id = self.session.host.subtask(method='tagBuild',
                                            arglist=[dest_tag, build_id, False, None, True],
                                            label='tag',
                                            parent=self.id,
                                            arch='noarch')
        self.wait(task_id)


class BaseBuildTask(BaseTaskHandler):
    """Base class for tasks the create a build root"""

    def checkHostArch(self, tag, hostdata, event=None):
        tagref = tag
        if isinstance(tag, dict):
            tagref = tag.get('id') or tag.get('name')
        opts = {}
        if event is not None:
            opts['event'] = event
        tag = self.session.getBuildConfig(tagref, **opts)
        if tag and tag['arches']:
            tag_arches = [koji.canonArch(a) for a in tag['arches'].split()]
            host_arches = hostdata['arches'].split()
            if not set(tag_arches).intersection(host_arches):
                self.logger.info('Task %s (%s): tag arches (%s) and '
                                 'host arches (%s) are disjoint' %
                                 (self.id, self.method,
                                  ', '.join(tag_arches), ', '.join(host_arches)))
                return False
        # otherwise...
        # This is in principle an error condition, but this is not a good place
        # to fail. Instead we proceed and let the task fail normally.
        return True


class BuildArchTask(BaseBuildTask):

    Methods = ['buildArch']

    def weight(self):
        return 1.5

    def updateWeight(self, name):
        """
        Update the weight of this task based on the package we're building.
        weight is scaled from a minimum of 1.5 to a maximum of 6, based on
        the average duration of a build of this package.
        """
        try:
            avg = self.session.getAverageBuildDuration(name, age=6)
        except koji.ParameterError:
            # for hub < 1.23
            avg = self.session.getAverageBuildDuration(name)

        if not avg:
            return
        if avg < 0:
            self.logger.warning("Negative average build duration for %s: %s", name, avg)
            return
        # increase the task weight by 0.75 for every hour of build duration
        adj = avg / 4800.0
        # cap the adjustment at +18 (24 hours)
        weight = self.weight() + min(18.0, adj)
        self.session.host.setTaskWeight(self.id, weight)

    def checkHost(self, hostdata):
        tag = self.params[1]
        return self.checkHostArch(tag, hostdata)

    def srpm_sanity_checks(self, filename):
        h_fields = koji.get_header_fields(filename, ['packager', 'vendor', 'distribution'])

        if not h_fields['packager']:
            raise koji.BuildError("The build system failed to set the packager tag")
        if not h_fields['vendor']:
            raise koji.BuildError("The build system failed to set the vendor tag")
        if not h_fields['distribution']:
            raise koji.BuildError("The build system failed to set the distribution tag")

    def handler(self, pkg, root, arch, keep_srpm, opts=None):
        """Build a package in a buildroot for one arch"""
        ret = {}
        if opts is None:
            opts = {}
        repo_id = opts.get('repo_id')
        if not repo_id:
            raise koji.BuildError("A repo id must be provided")
        repo_info = self.session.repoInfo(repo_id, strict=True)
        event_id = repo_info['create_event']

        # starting srpm should already have been uploaded by parent
        self.logger.debug("Reading SRPM")
        fn = self.localPath("work/%s" % pkg)
        if not os.path.exists(fn):
            raise koji.BuildError("SRPM file missing: %s" % fn)
        # peel E:N-V-R from package
        h = koji.get_rpm_header(fn)
        name = koji.get_header_field(h, 'name')
        if not koji.get_header_field(h, 'sourcepackage'):
            raise koji.BuildError("not a source package")
        # Disable checking for distribution in the initial SRPM because it
        # might have been built outside of the build system
        # if not koji.get_header_field(h, 'distribution'):
        #    raise koji.BuildError, "the distribution tag is not set in the original srpm"

        self.updateWeight(name)

        rootopts = {
            'repo_id': repo_id
        }
        if arch == "noarch":
            # There could have been forced taskarch Exclusive/ExcludeArch,
            # so we should honor it here.
            task = self.session.getTaskInfo(self.id)
            preferred_arch = task['arch']
        else:
            preferred_arch = None
        br_arch = self.find_arch(arch, self.session.host.getHost(),
                                 self.session.getBuildConfig(root, event=event_id),
                                 preferred_arch=preferred_arch)
        broot = BuildRoot(self.session, self.options, root, br_arch, self.id, **rootopts)
        broot.workdir = self.workdir

        self.logger.debug("Initializing buildroot")
        broot.init()

        # run build
        self.logger.debug("Running build")
        broot.build(fn, arch)

        # extract results
        resultdir = broot.resultdir()
        rpm_files = []
        srpm_files = []
        log_files = list(broot.logs)
        unexpected = []
        for f in os.listdir(resultdir):
            # files here should have one of two extensions: .log and .rpm
            if f[-4:] in (".log"):
                pass
                # should already be in log_files
            elif f[-8:] == ".src.rpm":
                srpm_files.append(f)
            elif f[-4:] == ".rpm":
                rpm_files.append(f)
            else:
                unexpected.append(f)

        # for noarch rpms compute rpmdiff hash
        rpmdiff_hash = {self.id: {}}
        for rpmf in rpm_files:
            if rpmf.endswith('.noarch.rpm'):
                fpath = os.path.join(resultdir, rpmf)
                d = koji.rpmdiff.Rpmdiff(fpath, fpath, ignore='S5TN')
                rpmdiff_hash[self.id][rpmf] = d.kojihash()
        if rpmdiff_hash[self.id]:
            log_name = 'noarch_rpmdiff.json'
            noarch_hash_path = os.path.join(broot.workdir, log_name)
            koji.dump_json(noarch_hash_path, rpmdiff_hash, indent=2, sort_keys=True)
            self.uploadFile(noarch_hash_path)
            log_files.append(log_name)

        self.logger.debug("rpms: %r" % rpm_files)
        self.logger.debug("srpms: %r" % srpm_files)
        self.logger.debug("logs: %r" % log_files)
        self.logger.debug("unexpected: %r" % unexpected)

        # upload files to storage server
        uploadpath = broot.getUploadPath()
        for f in rpm_files:
            self.uploadFile("%s/%s" % (resultdir, f))
        self.logger.debug("keep srpm %i %s %s" % (self.id, keep_srpm, opts))
        if keep_srpm:
            if len(srpm_files) == 0:
                raise koji.BuildError("no srpm files found for task %i" % self.id)
            if len(srpm_files) > 1:
                raise koji.BuildError("multiple srpm files found for task %i: %s" %
                                      (self.id, srpm_files))

            # Run sanity checks.  Any failures will throw a BuildError
            self.srpm_sanity_checks("%s/%s" % (resultdir, srpm_files[0]))

            self.logger.debug("uploading %s/%s to %s" % (resultdir, srpm_files[0], uploadpath))
            self.uploadFile("%s/%s" % (resultdir, srpm_files[0]))
        if rpm_files:
            ret['rpms'] = ["%s/%s" % (uploadpath, f) for f in rpm_files]
        else:
            ret['rpms'] = []
        if keep_srpm:
            ret['srpms'] = ["%s/%s" % (uploadpath, f) for f in srpm_files]
        else:
            ret['srpms'] = []
        ret['logs'] = ["%s/%s" % (uploadpath, f) for f in log_files]

        ret['brootid'] = broot.id

        broot.expire()
        # Let TaskManager clean up

        return ret


class MavenTask(MultiPlatformTask):

    Methods = ['maven']

    _taskWeight = 0.2

    def handler(self, url, target, opts=None):
        """Use Maven to build the source from the given url"""
        if opts is None:
            opts = {}
        self.opts = opts
        target_info = self.session.getBuildTarget(target)
        if not target_info:
            raise koji.BuildError('unknown build target: %s' % target)
        dest_tag = self.session.getTag(target_info['dest_tag'], strict=True)
        build_tag = self.session.getTag(target_info['build_tag'], strict=True)

        repo_id = opts.get('repo_id')
        if not repo_id:
            repo = self.session.getRepo(build_tag['id'])
            if repo:
                repo_id = repo['id']
            else:
                raise koji.BuildError('no repo for tag %s' % build_tag['name'])

        build_opts = dslice(opts, ['goals', 'profiles', 'properties', 'envs', 'patches',
                                   'packages', 'jvm_options', 'maven_options', 'deps', 'scratch'],
                            strict=False)
        build_opts['repo_id'] = repo_id

        self.build_task_id = self.session.host.subtask(method='buildMaven',
                                                       arglist=[url, build_tag, build_opts],
                                                       label='build',
                                                       parent=self.id,
                                                       arch='noarch')
        maven_results = self.wait(self.build_task_id)[self.build_task_id]
        maven_results['task_id'] = self.build_task_id

        build_info = None
        if not self.opts.get('scratch'):
            maven_info = maven_results['maven_info']
            if maven_info['version'].endswith('-SNAPSHOT'):
                raise koji.BuildError('-SNAPSHOT versions are only supported in scratch builds')
            build_info = koji.maven_info_to_nvr(maven_info)

            if not self.opts.get('skip_tag'):
                dest_cfg = self.session.getPackageConfig(dest_tag['id'], build_info['name'])
                # Make sure package is on the list for this tag
                if dest_cfg is None:
                    raise koji.BuildError("package %s not in list for tag %s"
                                          % (build_info['name'], dest_tag['name']))
                elif dest_cfg['blocked']:
                    raise koji.BuildError("package %s is blocked for tag %s"
                                          % (build_info['name'], dest_tag['name']))

            build_info = self.session.host.initMavenBuild(self.id, build_info, maven_info)
            self.build_id = build_info['id']

        try:
            rpm_results = None
            spec_url = self.opts.get('specfile')
            if spec_url:
                rpm_results = self.buildWrapperRPM(
                    spec_url, self.build_task_id, target_info, build_info, repo_id)

            if self.opts.get('scratch'):
                self.session.host.moveMavenBuildToScratch(self.id, maven_results, rpm_results)
            else:
                self.session.host.completeMavenBuild(
                    self.id, self.build_id, maven_results, rpm_results)
        except (SystemExit, ServerExit, KeyboardInterrupt):
            # we do not trap these
            raise
        except Exception:
            if not self.opts.get('scratch'):
                # scratch builds do not get imported
                self.session.host.failBuild(self.id, self.build_id)
            # reraise the exception
            raise

        if not self.opts.get('scratch') and not self.opts.get('skip_tag'):
            tag_task_id = self.session.host.subtask(method='tagBuild',
                                                    arglist=[dest_tag['id'],
                                                             self.build_id, False, None, True],
                                                    label='tag',
                                                    parent=self.id,
                                                    arch='noarch')
            self.wait(tag_task_id)


class BuildMavenTask(BaseBuildTask):

    Methods = ['buildMaven']

    _taskWeight = 1.5

    def _zip_dir(self, rootdir, filename):
        rootbase = os.path.basename(rootdir)
        roottrim = len(rootdir) - len(rootbase)
        zfo = zipfile.ZipFile(filename, 'w', zipfile.ZIP_DEFLATED)
        for dirpath, dirnames, filenames in os.walk(rootdir):
            for skip in ['CVS', '.svn', '.git']:
                if skip in dirnames:
                    dirnames.remove(skip)
            for filename in filenames:
                filepath = os.path.join(dirpath, filename)
                if os.path.islink(filepath):
                    content = os.readlink(filepath)
                    st = os.lstat(filepath)
                    mtime = time.localtime(st.st_mtime)
                    info = zipfile.ZipInfo(filepath[roottrim:])
                    info.external_attr |= 0o120000 << 16  # symlink file type
                    info.compress_type = zipfile.ZIP_STORED
                    info.date_time = mtime[:6]
                    zfo.writestr(info, content)
                else:
                    zfo.write(filepath, filepath[roottrim:])
        zfo.close()

    def checkHost(self, hostdata):
        tag = self.params[1]
        return self.checkHostArch(tag, hostdata)

    def handler(self, url, build_tag, opts=None):
        if opts is None:
            opts = {}
        self.opts = opts

        scm = SCM(url, allow_password=self.options.allow_password_in_scm_url)
        scm_policy_opts = {
            'user_id': self.taskinfo['owner'],
            'channel': self.session.getChannel(self.taskinfo['channel_id'],
                                               strict=True)['name'],
            'scratch': self.opts.get('scratch')
        }
        scm.assert_allowed(allowed=self.options.allowed_scms,
                           session=self.session,
                           by_config=self.options.allowed_scms_use_config,
                           by_policy=self.options.allowed_scms_use_policy,
                           policy_data=scm_policy_opts)
        repo_id = opts.get('repo_id')
        if not repo_id:
            raise koji.BuildError('A repo_id must be provided')
        repo_info = self.session.repoInfo(repo_id, strict=True)
        event_id = repo_info['create_event']

        br_arch = self.find_arch('noarch', self.session.host.getHost(
        ), self.session.getBuildConfig(build_tag['id'], event=event_id))
        maven_opts = opts.get('jvm_options')
        if not maven_opts:
            maven_opts = []
        for opt in maven_opts:
            if opt.startswith('-Xmx'):
                break
        else:
            # Give the JVM 2G to work with by default, if the build isn't specifying
            # its own max. memory
            maven_opts.append('-Xmx2048m')
        buildroot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id,
                              install_group='maven-build', setup_dns=True, repo_id=repo_id,
                              maven_opts=maven_opts, maven_envs=opts.get('envs'),
                              deps=opts.get('deps'))
        buildroot.workdir = self.workdir
        self.logger.debug("Initializing buildroot")
        buildroot.init()

        packages = opts.get('packages')
        if packages:
            rv = buildroot.mock(['--install'] + packages)
            self.session.host.setBuildRootState(buildroot.id, 'BUILDING')
            self.session.host.updateBuildRootList(buildroot.id, buildroot.getPackageList())
            if rv:
                buildroot.expire()
                raise koji.BuildrootError('error installing packages, %s' %
                                          buildroot._mockResult(rv, logfile='mock_output.log'))

        # existence of symlink should be sufficient
        if not os.path.lexists('%s/usr/bin/mvn' % buildroot.rootdir()):
            raise koji.BuildError('/usr/bin/mvn was not found in the buildroot')

        scmdir = '%s/maven/build' % buildroot.rootdir()
        outputdir = '%s/maven/output' % buildroot.rootdir()
        m2dir = '%s/builddir/.m2' % buildroot.rootdir()
        repodir = '%s/builddir/.m2/repository' % buildroot.rootdir()
        patchdir = '%s/maven/patches' % buildroot.rootdir()

        koji.ensuredir(scmdir)
        koji.ensuredir(outputdir)
        koji.ensuredir(repodir)
        koji.ensuredir(patchdir)

        logfile = self.workdir + '/checkout.log'
        uploadpath = self.getUploadDir()

        self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
                           build_tag=build_tag, scratch=opts.get('scratch'),
                           buildroot=buildroot)
        # Check out sources from the SCM
        sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile)
        self.run_callbacks("postSCMCheckout",
                           scminfo=scm.get_info(),
                           build_tag=build_tag,
                           scratch=opts.get('scratch'),
                           srcdir=sourcedir,
                           buildroot=buildroot)

        # zip up pristine sources for auditing purposes
        self._zip_dir(sourcedir, os.path.join(outputdir, 'scm-sources.zip'))

        # Checkout out patches, if present
        if self.opts.get('patches'):
            patchlog = self.workdir + '/patches.log'
            patch_scm = SCM(self.opts.get('patches'),
                            allow_password=self.options.allow_password_in_scm_url)
            patch_scm.assert_allowed(allowed=self.options.allowed_scms,
                                     session=self.session,
                                     by_config=self.options.allowed_scms_use_config,
                                     by_policy=self.options.allowed_scms_use_policy,
                                     policy_data=scm_policy_opts)
            self.run_callbacks('preSCMCheckout', scminfo=patch_scm.get_info(),
                               build_tag=build_tag, scratch=opts.get('scratch'),
                               buildroot=buildroot)
            # never try to check out a common/ dir when checking out patches
            patch_scm.use_common = False
            patchcheckoutdir = patch_scm.checkout(patchdir, self.session, uploadpath, patchlog)
            self.run_callbacks("postSCMCheckout",
                               scminfo=patch_scm.get_info(),
                               build_tag=build_tag,
                               scratch=opts.get('scratch'),
                               srcdir=patchcheckoutdir,
                               buildroot=buildroot)
            self._zip_dir(patchcheckoutdir, os.path.join(outputdir, 'patches.zip'))

        # Apply patches, if present
        if self.opts.get('patches'):
            # filter out directories and files beginning with . (probably scm metadata)
            patches = [patch for patch in os.listdir(patchcheckoutdir)
                       if os.path.isfile(os.path.join(patchcheckoutdir, patch)) and
                       patch.endswith('.patch')]
            if not patches:
                raise koji.BuildError('no patches found at %s' % self.opts.get('patches'))
            patches.sort()
            for patch in patches:
                cmd = ['/usr/bin/patch', '--verbose', '--no-backup-if-mismatch', '-d',
                       sourcedir, '-p1', '-i', os.path.join(patchcheckoutdir, patch)]
                ret = log_output(self.session, cmd[0], cmd,
                                 patchlog, uploadpath, logerror=1, append=1)
                if ret:
                    raise koji.BuildError(
                        'error applying patches from %s, see patches.log for details' %
                        self.opts.get('patches'))

        # Set ownership of the entire source tree to the mock user
        uid = pwd.getpwnam(self.options.mockuser)[2]
        gid = grp.getgrnam('mock')[2]
        self.chownTree(scmdir, uid, gid)
        self.chownTree(outputdir, uid, gid)
        self.chownTree(m2dir, uid, gid)
        if self.opts.get('patches'):
            self.chownTree(patchdir, uid, gid)

        settingsfile = '/builddir/.m2/settings.xml'
        buildroot.writeMavenSettings(settingsfile, outputdir)

        pomfile = 'pom.xml'
        maven_options = self.opts.get('maven_options', [])
        for i, opt in enumerate(maven_options):
            if opt == '-f' or opt == '--file':
                if len(maven_options) > (i + 1):
                    pomfile = maven_options[i + 1]
                    break
                else:
                    raise koji.BuildError('%s option requires a file path' % opt)
            elif opt.startswith('-f=') or opt.startswith('--file='):
                pomfile = opt.split('=', 1)[1]
                break
            elif opt.startswith('-f'):
                pomfile = opt[2:]
                break

        buildroot.mavenBuild(sourcedir, outputdir, repodir,
                             props=self.opts.get('properties'), profiles=self.opts.get('profiles'),
                             options=self.opts.get('maven_options'), goals=self.opts.get('goals'))

        build_pom = os.path.join(sourcedir, pomfile)
        if not os.path.exists(build_pom):
            raise koji.BuildError('%s does not exist' % pomfile)
        pom_info = koji.parse_pom(build_pom)
        maven_info = koji.pom_to_maven_info(pom_info)

        # give the zip files more descriptive names
        os.rename(os.path.join(outputdir, 'scm-sources.zip'),
                  os.path.join(outputdir, maven_info['artifact_id'] + '-' +
                               maven_info['version'] + '-scm-sources.zip'))
        if self.opts.get('patches'):
            os.rename(os.path.join(outputdir, 'patches.zip'),
                      os.path.join(outputdir, maven_info['artifact_id'] + '-' +
                                   maven_info['version'] + '-patches.zip'))

        logs = []
        output_files = {}

        for path, dirs, files in os.walk(outputdir):
            if not files:
                continue
            reldir = path[len(outputdir) + 1:]
            for filename in files:
                root, ext = os.path.splitext(filename)
                if ext == '.log':
                    logs.append(os.path.join(reldir, filename))
                else:
                    output_files.setdefault(reldir, []).append(filename)

        # upload the build output
        for filepath in logs:
            self.uploadFile(os.path.join(outputdir, filepath),
                            relPath=os.path.dirname(filepath))
        for relpath, files in six.iteritems(output_files):
            for filename in files:
                self.uploadFile(os.path.join(outputdir, relpath, filename),
                                relPath=relpath)

        # Also include the logs already uploaded by BuildRoot
        logs.extend(buildroot.logs)

        # Also include the checkout logs (uploaded during checkout)
        logs.append('checkout.log')
        if self.opts.get('patches'):
            logs.append('patches.log')

        buildroot.expire()

        return {'maven_info': maven_info,
                'buildroot_id': buildroot.id,
                'logs': logs,
                'files': output_files}


class WrapperRPMTask(BaseBuildTask):
    """Build a wrapper rpm around archives output from a Maven or Windows build.
    May either be called as a subtask or as a separate
    top-level task.  In the latter case it can either associate the new rpms
    with the existing build or create a new build."""

    Methods = ['wrapperRPM']

    _taskWeight = 1.5

    def copy_fields(self, src, tgt, *fields):
        for field in fields:
            tgt[field] = src.get(field)

    def spec_sanity_checks(self, filename):
        spec = koji._open_text_file(filename).read()
        for tag in ("Packager", "Distribution", "Vendor"):
            if re.match("%s:" % tag, spec, re.M):
                raise koji.BuildError("%s is not allowed to be set in spec file" % tag)
        for tag in ("packager", "distribution", "vendor"):
            if re.match(r"%%define\s+%s\s+" % tag, spec, re.M):
                raise koji.BuildError("%s is not allowed to be defined in spec file" % tag)

    def checkHost(self, hostdata):
        target = self.params[1]
        return self.checkHostArch(target['build_tag'], hostdata)

    def handler(self, spec_url, build_target, build, task, opts=None):
        if not opts:
            opts = {}

        if not (build or task):
            raise koji.BuildError('build and/or task must be specified')

        values = {}

        if build:
            maven_info = self.session.getMavenBuild(build['id'], strict=False)
            win_info = self.session.getWinBuild(build['id'], strict=False)
            image_info = self.session.getImageBuild(build['id'], strict=False)
        else:
            maven_info = None
            win_info = None
            image_info = None

        # list of artifact paths relative to kojiroot (not exposed to the specfile)
        artifact_relpaths = []
        # map of file extension to a list of files
        artifacts = {}
        # list of all files
        all_artifacts = []
        # list of all files with their repo path
        all_artifacts_with_path = []

        # makes generating relative paths easier
        self.pathinfo = koji.PathInfo(topdir='')

        if task:
            # called as a subtask of a build
            artifact_data = self.session.listTaskOutput(task['id'], all_volumes=True)

            for artifact_path in artifact_data:
                artifact_name = os.path.basename(artifact_path)
                base, ext = os.path.splitext(artifact_name)
                if ext == '.log':
                    # Exclude log files for consistency with the output of listArchives() used
                    # below
                    continue
                relpath = os.path.join(self.pathinfo.task(task['id']), artifact_path)[1:]
                for volume in artifact_data[artifact_path]:
                    volume_path = os.path.join(self.pathinfo.volumedir(volume), relpath)
                    artifact_relpaths.append(volume_path)
                    artifacts.setdefault(ext, []).append(artifact_name)
                    all_artifacts.append(artifact_name)
                    all_artifacts_with_path.append(volume_path)
        else:
            # called as a top-level task to create wrapper rpms for an existing build
            # verify that the build is complete
            if not build['state'] == koji.BUILD_STATES['COMPLETE']:
                raise koji.BuildError(
                    'cannot call wrapperRPM on a build that did not complete successfully')

            # get the list of files from the build instead of the task,
            # because the task output directory may have already been cleaned up
            if maven_info:
                build_artifacts = self.session.listArchives(buildID=build['id'], type='maven')
            elif win_info:
                build_artifacts = self.session.listArchives(buildID=build['id'], type='win')
            elif image_info:
                build_artifacts = self.session.listArchives(buildID=build['id'], type='image')
            else:
                raise koji.BuildError('unsupported build type')

            for artifact in build_artifacts:
                artifact_name = artifact['filename']
                base, ext = os.path.splitext(artifact_name)
                artifacts.setdefault(ext, []).append(artifact_name)
                all_artifacts.append(artifact_name)
                if ext == '.log':
                    # listArchives() should never return .log files, but we check for completeness
                    continue
                if maven_info:
                    repopath = self.pathinfo.mavenfile(artifact)
                    relpath = os.path.join(self.pathinfo.mavenbuild(build), repopath)[1:]
                    artifact_relpaths.append(relpath)
                    all_artifacts_with_path.append(repopath)
                elif win_info:
                    repopath = self.pathinfo.winfile(artifact)
                    relpath = os.path.join(self.pathinfo.winbuild(build), repopath)[1:]
                    artifact_relpaths.append(relpath)
                    all_artifacts_with_path.append(repopath)
                elif image_info:
                    ipath = self.pathinfo.imagebuild(build)
                    relpath = os.path.join(ipath, artifact_name)[1:]
                    artifact_relpaths.append(relpath)
                    all_artifacts_with_path.append(artifact_name)
                else:
                    # can't happen
                    assert False  # pragma: no cover

        if not artifacts:
            raise koji.BuildError('no output found for %s' % (
                task and koji.taskLabel(task) or koji.buildLabel(build)))

        values['artifacts'] = artifacts
        values['all_artifacts'] = all_artifacts
        values['all_artifacts_with_path'] = all_artifacts_with_path

        if build:
            self.copy_fields(build, values, 'epoch', 'name', 'version', 'release')
            if maven_info:
                values['maven_info'] = maven_info
            elif win_info:
                values['win_info'] = win_info
            elif image_info:
                values['image_info'] = image_info
            else:
                # can't happen
                assert False  # pragma: no cover
        else:
            task_result = self.session.getTaskResult(task['id'])
            if task['method'] == 'buildMaven':
                maven_info = task_result['maven_info']
                maven_nvr = koji.maven_info_to_nvr(maven_info)
                maven_nvr['release'] = '0.scratch'
                self.copy_fields(maven_nvr, values, 'epoch', 'name', 'version', 'release')
                values['maven_info'] = maven_info
            elif task['method'] == 'vmExec':
                self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release')
                values['win_info'] = {'platform': task_result['platform']}
            elif task['method'] in ('createLiveCD', 'createAppliance', 'createImage',
                                    'createLiveMedia'):
                self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release')
            else:
                # can't happen
                assert False  # pragma: no cover

        scm = SCM(spec_url, allow_password=self.options.allow_password_in_scm_url)
        scm.assert_allowed(allowed=self.options.allowed_scms,
                           session=self.session,
                           by_config=self.options.allowed_scms_use_config,
                           by_policy=self.options.allowed_scms_use_policy,
                           policy_data={
                               'user_id': self.taskinfo['owner'],
                               'channel': self.session.getChannel(self.taskinfo['channel_id'],
                                                                  strict=True)['name'],
                               'scratch': opts.get('scratch')
                           })

        if opts.get('create_build') and opts.get('custom_user_metadata'):
            try:
                json.dumps(opts['custom_user_metadata'])
            except TypeError:
                error_msg = 'custom_user_metadata is not JSON serializable'
                raise koji.BuildError(error_msg)

        # sort out remaining buildroot params
        build_tag = self.session.getTag(build_target['build_tag'], strict=True)
        repo_id = opts.get('repo_id')
        if not repo_id:
            # a parent task will generally pass in the repo it used, but if
            # we are top level, we'll need to find our own
            repo_info = self.getRepo(build_tag['id'], builds=opts.get('wait_builds'),
                                     wait=opts.get('wait_repo'))  # (subtask)
        else:
            repo_info = self.session.repoInfo(repo_id, strict=True)
        repo_id = repo_info['id']
        event_id = repo_info['create_event']
        br_arch = self.find_arch('noarch', self.session.host.getHost(
        ), self.session.getBuildConfig(build_tag['id'], event=event_id))

        buildroot = BuildRoot(self.session, self.options, build_tag['id'], br_arch, self.id,
                              install_group='wrapper-rpm-build', repo_id=repo_id)
        buildroot.workdir = self.workdir
        self.logger.debug("Initializing buildroot")
        buildroot.init()

        logfile = os.path.join(self.workdir, 'checkout.log')
        scmdir = buildroot.tmpdir() + '/scmroot'
        koji.ensuredir(scmdir)
        self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
                           build_tag=build_tag, scratch=opts.get('scratch'),
                           buildroot=buildroot)
        specdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile)
        self.run_callbacks("postSCMCheckout",
                           scminfo=scm.get_info(),
                           build_tag=build_tag,
                           scratch=opts.get('scratch'),
                           srcdir=specdir,
                           buildroot=buildroot)

        # get the source before chown, git > 2.35.2 would refuse to that later
        source = scm.get_source()
        spec_template = None
        for path, dir, files in os.walk(specdir):
            files.sort()
            for filename in files:
                if filename.endswith('.spec.tmpl'):
                    spec_template = os.path.join(path, filename)
                    break
        if not spec_template:
            raise koji.BuildError('no spec file template found at URL: %s' % spec_url)

        # Put the jars into the same directory as the specfile.  This directory will be
        # set to the rpm _sourcedir so other files in the SCM may be referenced in the
        # specfile as well.
        specdir = os.path.dirname(spec_template)
        for relpath in artifact_relpaths:
            localpath = self.localPath(relpath)
            # RPM requires all SOURCE files in the srpm to be in the same directory, so
            # we flatten any directory structure of the output files here.
            # If multiple files in the build have the same basename, duplicate files will
            # have their relative path prepended to their name, with / replaced with -.
            destpath = os.path.join(specdir, os.path.basename(relpath))
            if os.path.exists(destpath):
                destpath = os.path.join(specdir, relpath.replace('/', '-'))
            shutil.copy(localpath, destpath)

        # change directory to the specdir to the template can reference files there
        os.chdir(specdir)
        contents = Cheetah.Template.Template(file=spec_template,
                                             searchList=[values]).respond()
        contents = contents.encode('utf-8')

        specfile = spec_template[:-5]
        with open(specfile, 'wb') as specfd:
            specfd.write(contents)
        uploadpath = self.getUploadDir()
        self.session.uploadWrapper(specfile, uploadpath)

        # Run spec file sanity checks.  Any failures will throw a BuildError
        self.spec_sanity_checks(specfile)

        # chown the specdir to the mock user, because srpm creation happens
        # as an unprivileged user
        uid = pwd.getpwnam(self.options.mockuser)[2]
        gid = grp.getgrnam('mock')[2]
        self.chownTree(specdir, uid, gid)

        # build srpm
        self.logger.debug("Running srpm build")
        buildroot.build_srpm(specfile, specdir, None)

        srpms = glob.glob('%s/*.src.rpm' % buildroot.resultdir())
        if len(srpms) == 0:
            raise koji.BuildError('no srpms found in %s' % buildroot.resultdir())
        elif len(srpms) > 1:
            raise koji.BuildError('multiple srpms found in %s: %s' %
                                  (buildroot.resultdir(), ', '.join(srpms)))
        else:
            srpm = srpms[0]

        shutil.move(srpm, self.workdir)
        srpm = os.path.join(self.workdir, os.path.basename(srpm))

        self.new_build_id = None
        if opts.get('create_build') and not opts.get('scratch'):
            h = koji.get_rpm_header(srpm)
            data = koji.get_header_fields(h, ['name', 'version', 'release', 'epoch'])
            data['task_id'] = self.id
            data['source'] = source['source']
            data['extra'] = {'source': {'original_url': source['url']}}
            if opts.get('custom_user_metadata'):
                data['extra']['custom_user_metadata'] = opts['custom_user_metadata']
            # pass draft option in
            if opts.get('draft'):
                data['draft'] = opts.get('draft')
            self.logger.info("Reading package config for %(name)s" % data)
            pkg_cfg = self.session.getPackageConfig(build_target['dest_tag'], data['name'])
            if not opts.get('skip_tag'):
                # Make sure package is on the list for this tag
                if pkg_cfg is None:
                    raise koji.BuildError("package %s not in list for tag %s"
                                          % (data['name'], build_target['dest_tag_name']))
                elif pkg_cfg['blocked']:
                    raise koji.BuildError("package %s is blocked for tag %s"
                                          % (data['name'], build_target['dest_tag_name']))
            self.new_build_id = self.session.host.initBuild(data)

        try:
            buildroot.build(srpm)
        except (SystemExit, ServerExit, KeyboardInterrupt):
            raise
        except Exception:
            if self.new_build_id:
                self.session.host.failBuild(self.id, self.new_build_id)
                raise

        resultdir = buildroot.resultdir()
        srpm = None
        rpms = []
        specfile_name = os.path.basename(specfile)
        logs = ['checkout.log', specfile_name] + list(buildroot.logs)

        for filename in os.listdir(resultdir):
            if filename.endswith('.src.rpm'):
                if not srpm:
                    srpm = filename
                else:
                    if self.new_build_id:
                        self.session.host.failBuild(self.id, self.new_build_id)
                    raise koji.BuildError('multiple srpms found in %s: %s, %s' %
                                          (resultdir, srpm, filename))
            elif filename.endswith('.rpm'):
                rpms.append(filename)
            elif filename.endswith('.log'):
                pass
                # already included in buildroot.logs
            else:
                if self.new_build_id:
                    self.session.host.failBuild(self.id, self.new_build_id)
                raise koji.BuildError('unexpected file found in %s: %s' %
                                      (resultdir, filename))

        if not srpm:
            if self.new_build_id:
                self.session.host.failBuild(self.id, self.new_build_id)
            raise koji.BuildError('no srpm found')

        if not rpms:
            if self.new_build_id:
                self.session.host.failBuild(self.id, self.new_build_id)
            raise koji.BuildError('no rpms found')

        try:
            for rpm_fn in [srpm] + rpms:
                self.uploadFile(os.path.join(resultdir, rpm_fn))
        except (SystemExit, ServerExit, KeyboardInterrupt):
            raise
        except Exception:
            if self.new_build_id:
                self.session.host.failBuild(self.id, self.new_build_id)
                raise

        results = {'buildroot_id': buildroot.id,
                   'srpm': srpm,
                   'rpms': rpms,
                   'logs': logs,
                   'source': source}
        if opts.get('create_build') and opts.get('custom_user_metadata'):
            results['custom_user_metadata'] = opts['custom_user_metadata']

        if not task:
            # Called as a standalone top-level task, so handle the rpms now.
            # Otherwise we let the parent task handle it.
            uploaddir = self.getUploadDir()
            relsrpm = uploaddir + '/' + srpm
            relrpms = [uploaddir + '/' + rpm for rpm in rpms]
            rellogs = [uploaddir + '/' + log for log in logs]
            if opts.get('scratch'):
                self.session.host.moveBuildToScratch(
                    self.id, relsrpm, relrpms, {'noarch': rellogs})
            else:
                if opts.get('create_build'):
                    brmap = dict.fromkeys([relsrpm] + relrpms, buildroot.id)
                    try:
                        self.session.host.completeBuild(self.id, self.new_build_id,
                                                        relsrpm, relrpms, brmap,
                                                        {'noarch': rellogs})
                    except (SystemExit, ServerExit, KeyboardInterrupt):
                        raise
                    except Exception:
                        self.session.host.failBuild(self.id, self.new_build_id)
                        raise
                    if not opts.get('skip_tag'):
                        tag_task_id = self.session.host.subtask(method='tagBuild',
                                                                arglist=[build_target['dest_tag'],
                                                                         self.new_build_id, False,
                                                                         None, True],
                                                                label='tag', parent=self.id,
                                                                arch='noarch')
                        self.wait(tag_task_id)
                else:
                    self.session.host.importWrapperRPMs(self.id, build['id'], results)

        # no need to upload logs, they've already been streamed to the hub
        # during the build process

        buildroot.expire()

        return results


class ChainMavenTask(MultiPlatformTask):

    Methods = ['chainmaven']

    _taskWeight = 0.2

    def handler(self, builds, target, opts=None):
        """Run a sequence of Maven builds in dependency order"""
        if not opts:
            opts = {}
        target_info = self.session.getBuildTarget(target)
        if not target_info:
            raise koji.BuildError('unknown build target: %s' % target)
        dest_tag = self.session.getTag(target_info['dest_tag'], strict=True)

        if not (opts.get('scratch') or opts.get('skip_tag')):
            for package in builds:
                dest_cfg = self.session.getPackageConfig(dest_tag['id'], package)
                # Make sure package is on the list for this tag
                if dest_cfg is None:
                    raise koji.BuildError("package %s not in list for tag %s"
                                          % (package, dest_tag['name']))
                elif dest_cfg['blocked']:
                    raise koji.BuildError("package %s is blocked for tag %s"
                                          % (package, dest_tag['name']))

        self.depmap = {}
        for package, params in builds.items():
            self.depmap[package] = set(params.get('buildrequires', []))

        todo = copy.deepcopy(self.depmap)
        running = {}
        self.done = {}
        self.results = []

        while True:
            ready = [package for package, deps in todo.items() if not deps]
            if not ready and not running:
                break
            for package in ready:
                params = builds[package]
                buildtype = params.get('type', 'maven')
                task_url = params['scmurl']
                task_opts = dslice_ex(params, ['scmurl', 'buildrequires', 'type'], strict=False)
                if buildtype == 'maven':
                    task_deps = list(self.depset(package))
                    if task_deps:
                        task_opts['deps'] = task_deps

                if not opts.get('force'):
                    # check for a duplicate build (a build performed with the
                    # same scmurl and options)
                    dup_build = self.get_duplicate_build(
                        dest_tag['name'], package, params, task_opts)
                    # if we find one, mark the package as built and remove it from todo
                    if dup_build:
                        self.done[package] = dup_build['nvr']
                        for deps in todo.values():
                            deps.discard(package)
                        del todo[package]
                        self.results.append('%s previously built from %s' %
                                            (dup_build['nvr'], task_url))
                        continue
                task_opts.update(dslice(opts, ['skip_tag', 'scratch'], strict=False))

                if buildtype == 'maven':
                    if opts.get('debug'):
                        task_opts.setdefault('maven_options', []).append('--debug')
                    task_id = self.subtask('maven', [task_url, target, task_opts],
                                           label=package)
                elif buildtype == 'wrapper':
                    pkg_to_wrap = params['buildrequires'][0]
                    to_wrap = self.done[pkg_to_wrap]

                    if isinstance(to_wrap, six.integer_types):
                        task_to_wrap = self.session.getTaskInfo(to_wrap, request=True)
                        build_to_wrap = None
                    else:
                        build_to_wrap = self.session.getBuild(to_wrap, strict=True)
                        task_to_wrap = None
                    target_info = self.session.getBuildTarget(target, strict=True)
                    repo_info = self.getRepo(target_info['build_tag'])
                    task_opts['repo_id'] = repo_info['id']
                    task_id = self.subtask('wrapperRPM', [task_url, target_info,
                                                          build_to_wrap, task_to_wrap,
                                                          task_opts],
                                           label=package)
                else:
                    raise koji.BuildError('unsupported build type: %s' % buildtype)

                running[task_id] = package
                del todo[package]
            try:
                results = self.wait(to_list(running.keys()))
            except (six.moves.xmlrpc_client.Fault, koji.GenericError):
                # One task has failed, wait for the rest to complete before the
                # chainmaven task fails.  self.wait(all=True) should thrown an exception.
                self.wait(all=True)
                raise
            # if we get here, results is a map whose keys are the ids of tasks
            # that have completed successfully
            for task_id in results:
                package = running.pop(task_id)
                task_url = builds[package]['scmurl']
                if opts.get('scratch'):
                    if builds[package].get('type') == 'wrapper':
                        self.done[package] = task_id
                    else:
                        children = self.session.getTaskChildren(task_id)
                        for child in children:
                            # we want the ID of the buildMaven task because the
                            # output dir of that task is where the Maven repo is
                            if child['method'] == 'buildMaven':
                                self.done[package] = child['id']
                                break
                        else:
                            raise koji.BuildError(
                                'could not find buildMaven subtask of %s' % task_id)
                    self.results.append('%s built from %s by task %s' %
                                        (package, task_url, task_id))
                else:
                    task_builds = self.session.listBuilds(taskID=task_id)
                    if not task_builds:
                        raise koji.BuildError('could not find build for task %s' % task_id)
                    task_build = task_builds[0]
                    self.done[package] = task_build['nvr']
                    self.results.append('%s built from %s' % (task_build['nvr'], task_url))
                for deps in todo.values():
                    deps.discard(package)

        if todo:
            # should never happen, the client should have checked for circular dependencies
            raise koji.BuildError('unable to run chain build, circular dependencies')
        return self.results

    def depset(self, package):
        deps = set()
        for dep in self.depmap[package]:
            deps.add(self.done[dep])
            deps.update(self.depset(dep))
        return deps

    def dicts_equal(self, a, b):
        """Check if two dicts are equal.  They are considered equal if they
        have the same keys and those keys have the same values.  If a value is
        list, it will be considered equal to a list with the same values in
        a different order."""
        akeys = to_list(a.keys())
        bkeys = to_list(b.keys())
        if sorted(akeys) != sorted(bkeys):
            return False
        for key in akeys:
            aval = a.get(key)
            bval = b.get(key)
            if not isinstance(aval, type(bval)):
                return False
            if isinstance(aval, dict):
                if not self.dicts_equal(aval, bval):
                    return False
            elif isinstance(aval, list):
                if not sorted(aval) == sorted(bval):
                    return False
            else:
                if not aval == bval:
                    return False
        return True

    def get_duplicate_build(self, tag, package, params, task_opts):
        """Find the latest build of package in tag and compare it to the
        scmurl and task_opts.  If they're identical, return the build."""
        builds = self.session.getLatestBuilds(tag, package=package)
        if not builds:
            return None
        build = builds[0]
        if not build['task_id']:
            return None
        build_task = self.session.getTaskInfo(build['task_id'], request=True)
        request = build_task['request']
        if request[0] != params['scmurl']:
            return None
        if params.get('type') == 'wrapper':
            wrapped_build = request[2]
            pkg_to_wrap = params['buildrequires'][0]
            nvr_to_wrap = self.done[pkg_to_wrap]
            if wrapped_build['nvr'] != nvr_to_wrap:
                return None
            # For a wrapper-rpm build, the only parameters that really matter
            # are the scmurl and the wrapped NVR.  These both match, so
            # return the existing build.
            return build
        if len(request) > 2:
            build_opts = dslice_ex(request[2], ['skip_tag', 'scratch'], strict=False)
        else:
            build_opts = {}
        task_opts = copy.deepcopy(task_opts)
        # filter out options that don't affect the build output
        # to avoid unnecessary rebuilds
        for opts in [build_opts, task_opts]:
            if 'maven_options' in opts:
                maven_options = opts['maven_options']
                for opt in ['-e', '--errors', '-q', '--quiet',
                            '-V', '--show-version', '-X', '--debug']:
                    if opt in maven_options:
                        maven_options.remove(opt)
                if not maven_options:
                    del opts['maven_options']
            if 'jvm_options' in opts:
                del opts['jvm_options']
        if not self.dicts_equal(build_opts, task_opts):
            return None
        # everything matches
        return build


class TagBuildTask(BaseTaskHandler):

    Methods = ['tagBuild']
    # XXX - set weight?

    def handler(self, tag_id, build_id, force=False, fromtag=None, ignore_success=False):
        task = self.session.getTaskInfo(self.id)
        user_id = task['owner']
        try:
            self.session.getBuild(build_id, strict=True)
            self.session.getTag(tag_id, strict=True)

            # several basic sanity checks have already been run (and will be run
            # again when we make the final call). Our job is to perform the more
            # computationally expensive 'post' tests.

            # XXX - add more post tests
            self.session.host.tagBuild(self.id, tag_id, build_id, force=force, fromtag=fromtag)
            self.session.host.tagNotification(
                True, tag_id, fromtag, build_id, user_id, ignore_success)
        except Exception as e:
            exctype, value = sys.exc_info()[:2]
            self.session.host.tagNotification(
                False, tag_id, fromtag, build_id, user_id, ignore_success, "%s: %s" %
                                                                           (exctype, value))
            raise e


class BuildImageTask(MultiPlatformTask):

    def initImageBuild(self, name, version, release, target_info, opts):
        """create a build object for this image build"""
        pkg_cfg = self.session.getPackageConfig(target_info['dest_tag_name'],
                                                name)
        self.logger.debug("%r" % pkg_cfg)
        if not opts.get('skip_tag') and not opts.get('scratch'):
            # Make sure package is on the list for this tag
            if pkg_cfg is None:
                raise koji.BuildError("package (image) %s not in list for tag %s" %
                                      (name, target_info['dest_tag_name']))
            elif pkg_cfg['blocked']:
                raise koji.BuildError("package (image)  %s is blocked for tag %s" %
                                      (name, target_info['dest_tag_name']))
        return self.session.host.initImageBuild(self.id,
                                                dict(name=name, version=version, release=release,
                                                     epoch=0))


class BuildBaseImageTask(BuildImageTask):
    Methods = ['image']

    def handler(self, name, version, arches, target, inst_tree, opts=None):
        """Governing task for building an appliance using Oz"""
        target_info = self.session.getBuildTarget(target, strict=True)
        build_tag = target_info['build_tag']
        repo_info = self.getRepo(build_tag)
        # check requested arches against build tag
        buildconfig = self.session.getBuildConfig(build_tag)
        if not buildconfig['arches']:
            raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
        tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
        for arch in arches:
            if koji.canonArch(arch) not in tag_archlist:
                raise koji.BuildError("Invalid arch for build tag: %s" % arch)

        if not opts:
            opts = {}

        if not ozif_enabled:
            self.logger.error(
                "ImageFactory features require the following dependencies: pykickstart, "
                "imagefactory, oz and possibly python-hashlib")
            raise RefuseTask('ImageFactory functions not available')

        # Policy check
        task_info = self.session.getTaskInfo(self.id)
        policy_data = {
            'user_id': task_info['owner'],
            'source': opts.get('ksurl'),
            'task_id': self.id,
            'build_tag': build_tag,  # id
            'skip_tag': bool(self.opts.get('skip_tag')),
            'scratch': bool(opts.get('scratch')),
            'from_scm': False,
            'repo_id': opts.get('repo_id'),
            'target': target_info['name'],
        }
        if not self.opts.get('skip_tag'):
            policy_data['tag'] = target_info['dest_tag']  # id
        self.session.host.assertPolicy('build_rpm', policy_data)

        # build image(s)
        bld_info = None
        try:
            release = opts.get('release')
            if '-' in version:
                raise koji.ApplianceError('The Version may not have a hyphen')
            if release and '-' in release:
                raise koji.ApplianceError('The Release may not have a hyphen')
            if not opts.get('scratch'):
                bld_info = self.initImageBuild(name, version, release,
                                               target_info, opts)
                release = bld_info['release']
            elif not release:
                # scratch build should have some reasonable release
                release = self.session.getNextRelease(dict(name=name, version=version))

            subtasks = {}
            self.logger.debug("Spawning jobs for image arches: %r" % (arches))
            canfail = []
            for arch in arches:
                inst_url = inst_tree.replace('$arch', arch)
                subtasks[arch] = self.session.host.subtask(
                    method='createImage',
                    arglist=[name, version, release, arch, target_info,
                             build_tag, repo_info, inst_url, opts],
                    label=arch, parent=self.id, arch=arch)
                if arch in opts.get('optional_arches', []):
                    canfail.append(subtasks[arch])
            self.logger.debug("Got image subtasks: %r" % (subtasks))
            self.logger.debug("Waiting on image subtasks (%s can fail)..." % canfail)
            results = self.wait(to_list(subtasks.values()), all=True,
                                failany=True, canfail=canfail)

            # if everything failed, fail even if all subtasks are in canfail
            self.logger.debug('subtask results: %r', results)
            all_failed = True
            for result in results.values():
                if not isinstance(result, dict) or 'faultCode' not in result:
                    all_failed = False
                    break
            if all_failed:
                raise koji.GenericError("all subtasks failed")

            # determine ignored arch failures
            ignored_arches = set()
            for arch in arches:
                if arch in opts.get('optional_arches', []):
                    task_id = subtasks[arch]
                    result = results[task_id]
                    if isinstance(result, dict) and 'faultCode' in result:
                        ignored_arches.add(arch)

            # wrap in an RPM if asked
            spec_url = opts.get('specfile')
            for arch in arches:
                # get around an xmlrpc limitation, use arches for keys instead
                results[arch] = results[subtasks[arch]]
                del results[subtasks[arch]]
                if spec_url and arch not in ignored_arches:
                    subtask = subtasks[arch]
                    results[arch]['rpmresults'] = self.buildWrapperRPM(
                        spec_url, subtask, target_info, bld_info,
                        repo_info['id'])

            # make sure we only import the user-submitted kickstart file one
            # time, otherwise we will have collisions. Remove it from exactly
            # 1 results hash from the subtasks
            if 'kickstart' in opts:
                saw_ks = False
                for arch in results:
                    if arch in ignored_arches:
                        continue
                    ks = os.path.basename(opts['kickstart'])
                    if ks in results[arch]['files']:
                        if saw_ks:
                            results[arch]['files'].remove(ks)
                        saw_ks = True

            self.logger.debug('Image Results for hub: %s' % results)
            if opts.get('scratch'):
                self.session.host.moveImageBuildToScratch(self.id, results)
            else:
                self.session.host.completeImageBuild(self.id, bld_info['id'],
                                                     results)

        except (SystemExit, ServerExit, KeyboardInterrupt):
            # we do not trap these
            raise
        except Exception:
            if not opts.get('scratch'):
                # scratch builds do not get imported
                if bld_info:
                    self.session.host.failBuild(self.id, bld_info['id'])
            # reraise the exception
            raise

        # tag it
        if not opts.get('scratch') and not opts.get('skip_tag'):
            tag_task_id = self.session.host.subtask(method='tagBuild',
                                                    arglist=[target_info['dest_tag'],
                                                             bld_info['id'], False, None, True],
                                                    label='tag', parent=self.id, arch='noarch')
            self.wait(tag_task_id)

        # report results
        report = ''
        if opts.get('scratch'):
            respath = ', '.join(
                [os.path.join(koji.pathinfo.work(),
                              koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()])
            report += 'Scratch '
        else:
            respath = koji.pathinfo.imagebuild(bld_info)
        report += 'image build results in: %s' % respath
        return report


class BuildApplianceTask(BuildImageTask):
    Methods = ['appliance']

    def handler(self, name, version, arch, target, ksfile, opts=None):
        """Governing task for building an appliance"""
        target_info = self.session.getBuildTarget(target, strict=True)
        build_tag = target_info['build_tag']
        repo_info = self.getRepo(build_tag)
        # check requested arch against build tag
        buildconfig = self.session.getBuildConfig(build_tag)
        if not buildconfig['arches']:
            raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
        tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
        if koji.canonArch(arch) not in tag_archlist:
            raise koji.BuildError("Invalid arch for build tag: %s" % arch)

        if not opts:
            opts = {}

        if not image_enabled:
            self.logger.error(
                "Appliance features require the following dependencies: "
                "pykickstart, and possibly python-hashlib")
            raise RefuseTask('Appliance functions not available')

        # build image
        bld_info = None
        try:
            release = opts.get('release')
            if not opts.get('scratch'):
                bld_info = self.initImageBuild(name, version, release,
                                               target_info, opts)
                release = bld_info['release']
            create_task_id = self.session.host.subtask(method='createAppliance',
                                                       arglist=[name, version, release, arch,
                                                                target_info, build_tag,
                                                                repo_info, ksfile, opts],
                                                       label='appliance', parent=self.id,
                                                       arch=arch)
            results = self.wait(create_task_id)
            self.logger.info('image build task (%s) completed' % create_task_id)
            self.logger.info('results: %s' % results)

            # wrap in an RPM if asked
            spec_url = opts.get('specfile')
            if spec_url:
                results[create_task_id]['rpmresults'] = self.buildWrapperRPM(
                    spec_url, create_task_id,
                    target_info, bld_info, repo_info['id'])
            results[str(create_task_id)] = results[create_task_id]
            del results[create_task_id]

            # import the image (move it too)
            if not opts.get('scratch'):
                self.session.host.completeImageBuild(self.id, bld_info['id'], results)
            else:
                self.session.host.moveImageBuildToScratch(self.id, results)

        except (SystemExit, ServerExit, KeyboardInterrupt):
            # we do not trap these
            raise
        except Exception:
            if not opts.get('scratch'):
                # scratch builds do not get imported
                if bld_info:
                    self.session.host.failBuild(self.id, bld_info['id'])
            # reraise the exception
            raise

        # tag it
        if not opts.get('scratch') and not opts.get('skip_tag'):
            tag_task_id = self.session.host.subtask(method='tagBuild',
                                                    arglist=[target_info['dest_tag'],
                                                             bld_info['id'], False, None, True],
                                                    label='tag', parent=self.id, arch='noarch')
            self.wait(tag_task_id)

        # report results
        if opts.get('scratch'):
            respath = os.path.join(koji.pathinfo.work(),
                                   koji.pathinfo.taskrelpath(create_task_id))
            report = 'Scratch '
        else:
            respath = koji.pathinfo.imagebuild(bld_info)
            report = ''
        report += 'appliance build results in: %s' % respath
        return report


class BuildLiveCDTask(BuildImageTask):
    Methods = ['livecd']

    def handler(self, name, version, arch, target, ksfile, opts=None):
        """Governing task for building LiveCDs"""
        target_info = self.session.getBuildTarget(target, strict=True)
        build_tag = target_info['build_tag']
        repo_info = self.getRepo(build_tag)
        # check requested arch against build tag
        buildconfig = self.session.getBuildConfig(build_tag)
        if not buildconfig['arches']:
            raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
        tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]
        if koji.canonArch(arch) not in tag_archlist:
            raise koji.BuildError("Invalid arch for build tag: %s" % arch)

        if not opts:
            opts = {}
        if not image_enabled:
            self.logger.error("LiveCD features require the following dependencies: "
                              "pykickstart, pycdio, and possibly python-hashlib")
            raise RefuseTask('LiveCD functions not available')

        # build the image
        bld_info = None
        try:
            if opts.get('release') is None:
                release = self.session.getNextRelease({'name': name, 'version': version})
            else:
                release = opts.get('release')
            if not opts.get('scratch'):
                bld_info = self.initImageBuild(name, version, release,
                                               target_info, opts)
                release = bld_info['release']
            create_task_id = self.session.host.subtask(method='createLiveCD',
                                                       arglist=[name, version, release, arch,
                                                                target_info, build_tag,
                                                                repo_info, ksfile, opts],
                                                       label='livecd', parent=self.id, arch=arch)
            results = self.wait(create_task_id)
            self.logger.info('image build task (%s) completed' % create_task_id)
            self.logger.info('results: %s' % results)

            # wrap in an RPM if needed
            spec_url = opts.get('specfile')
            if spec_url:
                results[create_task_id]['rpmresults'] = self.buildWrapperRPM(
                    spec_url, create_task_id,
                    target_info, bld_info, repo_info['id'])
            results[str(create_task_id)] = results[create_task_id]
            del results[create_task_id]

            # import it (and move)
            if not opts.get('scratch'):
                self.session.host.completeImageBuild(self.id, bld_info['id'], results)
            else:
                self.session.host.moveImageBuildToScratch(self.id, results)

        except (SystemExit, ServerExit, KeyboardInterrupt):
            # we do not trap these
            raise
        except Exception:
            if not opts.get('scratch'):
                # scratch builds do not get imported
                if bld_info:
                    self.session.host.failBuild(self.id, bld_info['id'])
            # reraise the exception
            raise

        # tag it if necessary
        if not opts.get('scratch') and not opts.get('skip_tag'):
            tag_task_id = self.session.host.subtask(method='tagBuild',
                                                    arglist=[target_info['dest_tag'],
                                                             bld_info['id'], False, None, True],
                                                    label='tag', parent=self.id, arch='noarch')
            self.wait(tag_task_id)

        # report the results
        if opts.get('scratch'):
            respath = os.path.join(koji.pathinfo.work(),
                                   koji.pathinfo.taskrelpath(create_task_id))
            report = 'Scratch '
        else:
            respath = koji.pathinfo.imagebuild(bld_info)
            report = ''
        report += 'livecd build results in: %s' % respath
        return report


class BuildLiveMediaTask(BuildImageTask):
    Methods = ['livemedia']

    def handler(self, name, version, arches, target, ksfile, opts=None):
        """Governing task for building live media"""

        target_info = self.session.getBuildTarget(target, strict=True)
        build_tag = target_info['build_tag']
        repo_info = self.getRepo(build_tag)
        # check requested arch against build tag
        buildconfig = self.session.getBuildConfig(build_tag)
        if not buildconfig['arches']:
            raise koji.BuildError("No arches for tag %(name)s [%(id)s]" % buildconfig)
        tag_archlist = [koji.canonArch(a) for a in buildconfig['arches'].split()]

        # check arches and remove duplicates
        arches = set(arches)
        for arch in arches:
            if koji.canonArch(arch) not in tag_archlist:
                raise koji.BuildError("Invalid arch for build tag: %s" % arch)

        if not opts:
            opts = {}
        if not image_enabled:
            # XXX - are these still required here?
            self.logger.error("Missing the following dependencies: "
                              "pykickstart, pycdio, and possibly python-hashlib")
            raise RefuseTask('Live Media functions not available')

        # build the image
        bld_info = None
        try:
            release = opts.get('release')
            if not opts.get('scratch'):
                bld_info = self.initImageBuild(name, version, release,
                                               target_info, opts)
                release = bld_info['release']
            subtasks = {}
            canfail = []
            for arch in arches:
                subtasks[arch] = self.subtask('createLiveMedia',
                                              [name, version, release,
                                               arch, target_info,
                                               build_tag, repo_info, ksfile, opts],
                                              label='livemedia %s' % arch, arch=arch)
                if arch in opts.get('optional_arches', []):
                    canfail.append(subtasks[arch])
            self.logger.debug("Tasks that can fail: %r", canfail)

            self.logger.debug("Got image subtasks: %r", subtasks)
            self.logger.debug("Waiting on livemedia subtasks...")
            results = self.wait(to_list(subtasks.values()), all=True,
                                failany=True, canfail=canfail)

            # if everything failed, fail even if all subtasks are in canfail
            self.logger.debug('subtask results: %r', results)
            all_failed = True
            for result in results.values():
                if not isinstance(result, dict) or 'faultCode' not in result:
                    all_failed = False
                    break
            if all_failed:
                raise koji.GenericError("all subtasks failed")

            # determine ignored arch failures
            ignored_arches = set()
            for arch in arches:
                if arch in opts.get('optional_arches', []):
                    task_id = subtasks[arch]
                    result = results[task_id]
                    if isinstance(result, dict) and 'faultCode' in result:
                        ignored_arches.add(arch)

            # wrap each image an RPM if needed
            spec_url = opts.get('specfile')
            if spec_url:
                wrapper_tasks = {}
                for arch in arches:
                    subtask_id = subtasks[arch]
                    result = results[subtask_id]
                    tinfo = self.session.getTaskInfo(subtask_id)
                    if arch in ignored_arches:
                        continue
                    arglist = [spec_url, target_info, bld_info, tinfo,
                               {'repo_id': repo_info['id']}]
                    wrapper_tasks[arch] = self.subtask('wrapperRPM', arglist,
                                                       label='wrapper %s' % arch, arch='noarch')

                results2 = self.wait(to_list(wrapper_tasks.values()), all=True, failany=True)
                self.logger.debug('wrapper results: %r', results2)

                # add wrapper rpm results into main results
                for arch in arches:
                    if arch in ignored_arches:
                        continue
                    result = results[subtasks[arch]]
                    result2 = results2[wrapper_tasks[arch]]
                    result['rpmresults'] = result2

            # re-key results for xmlrpc friendliness
            results = dict([(str(k), results[k]) for k in results])

            # import it (and move)
            if not opts.get('scratch'):
                self.session.host.completeImageBuild(self.id, bld_info['id'], results)
            else:
                self.session.host.moveImageBuildToScratch(self.id, results)

        except (SystemExit, ServerExit, KeyboardInterrupt):
            # we do not trap these
            raise
        except Exception:
            if not opts.get('scratch'):
                # scratch builds do not get imported
                if bld_info:
                    self.session.host.failBuild(self.id, bld_info['id'])
            # reraise the exception
            raise

        # tag it if necessary
        if not opts.get('scratch') and not opts.get('skip_tag'):
            tag_task_id = self.session.host.subtask(method='tagBuild',
                                                    arglist=[target_info['dest_tag'],
                                                             bld_info['id'], False, None, True],
                                                    label='tag', parent=self.id, arch='noarch')
            self.wait(tag_task_id)

        # report the results
        if opts.get('scratch'):
            respath = ', '.join(
                [os.path.join(koji.pathinfo.work(),
                              koji.pathinfo.taskrelpath(tid)) for tid in subtasks.values()])
            report = 'Scratch '
        else:
            respath = koji.pathinfo.imagebuild(bld_info)
            report = ''
        report += 'livecd build results in: %s' % respath
        return report


# A generic task for building cd or disk images using chroot-based tools.
# Other chroot-based image handlers should inherit this.
class ImageTask(BaseTaskHandler):
    Methods = []
    # default to bind mounting /dev, but allow subclasses to change
    # this
    bind_opts = {'dirs': {'/dev': '/dev', }}

    def makeImgBuildRoot(self, buildtag, repoinfo, arch, inst_group):
        """
        Create and prepare the chroot we're going to build an image in.
        Binds necessary directories and creates needed device files.

        @args:
            buildtag: a build tag
            repoinfo: a session.getRepo() object
            arch: a canonical architecture name
            inst_group: a string representing the yum group to install with
        @returns: a buildroot object
        """

        rootopts = {'install_group': inst_group,
                    'setup_dns': True,
                    'repo_id': repoinfo['id']}
        if self.bind_opts:
            rootopts['bind_opts'] = self.bind_opts

        broot = BuildRoot(self.session, self.options, buildtag, arch, self.id, **rootopts)
        broot.workdir = self.workdir

        # create the mock chroot
        self.logger.debug("Initializing image buildroot")
        broot.init()
        self.logger.debug("Image buildroot ready: " + broot.rootdir())
        return broot

    def fetchKickstart(self, broot, ksfile, build_tag):
        """
        Retrieve the kickstart file we were given (locally or remotely) and
        upload it.

        Note that if the KS file existed locally, then "ksfile" is a relative
        path to it in the /mnt/koji/work directory. If not, then it is still
        the parameter the user passed in initially, and we assume it is a
        relative path in a remote scm. The user should have passed in an scm
        url with --ksurl.

        @args:
            broot: a buildroot object
            ksfile: path to a kickstart file
            build_tag: build tag name
        @returns: absolute path to the retrieved kickstart file
        """
        scmdir = broot.tmpdir()
        koji.ensuredir(scmdir)
        self.logger.debug("ksfile = %s" % ksfile)
        if self.opts.get('ksurl'):
            scm = SCM(self.opts['ksurl'], allow_password=self.options.allow_password_in_scm_url)
            scm.assert_allowed(allowed=self.options.allowed_scms,
                               session=self.session,
                               by_config=self.options.allowed_scms_use_config,
                               by_policy=self.options.allowed_scms_use_policy,
                               policy_data={
                                   'user_id': self.taskinfo['owner'],
                                   'channel': self.session.getChannel(self.taskinfo['channel_id'],
                                                                      strict=True)['name'],
                                   'scratch': self.opts.get('scratch')
                               })
            logfile = os.path.join(self.workdir, 'checkout.log')
            self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
                               build_tag=build_tag, scratch=self.opts.get('scratch'),
                               buildroot=broot)
            scmsrcdir = scm.checkout(scmdir, self.session, self.getUploadDir(), logfile)
            self.run_callbacks("postSCMCheckout",
                               scminfo=scm.get_info(),
                               build_tag=build_tag,
                               scratch=self.opts.get('scratch'),
                               srcdir=scmsrcdir,
                               buildroot=broot)
            kspath = os.path.join(scmsrcdir, ksfile)
        else:
            kspath = self.localPath("work/%s" % ksfile)

        if not os.path.exists(kspath):
            raise koji.BuildError('Kickstart %s does not exist.' % kspath)

        self.uploadFile(kspath)  # upload the original ks file
        return kspath  # full absolute path to the file in the chroot

    def readKickstart(self, kspath, opts):
        """
        Read a kickstart file and save the ks object as a task member.

        @args:
            kspath: path to a kickstart file
        @returns: None
        """
        # XXX: If the ks file came from a local path and has %include
        #      macros, *-creator will fail because the included
        #      kickstarts were not copied into the chroot. For now we
        #      require users to flatten their kickstart file if submitting
        #      the task with a local path.
        #
        #      Note that if an SCM URL was used instead, %include macros
        #      may not be a problem if the included kickstarts are present
        #      in the repository we checked out.
        if opts.get('ksversion'):
            version = ksparser.version.makeVersion(opts['ksversion'])
        else:
            version = ksparser.version.makeVersion()
        self.ks = ksparser.KickstartParser(version)
        try:
            self.ks.readKickstart(kspath)
        except IOError as e:
            raise koji.LiveCDError("Failed to read kickstart file "
                                   "'%s' : %s" % (kspath, e))
        except kserrors.KickstartError as e:
            raise koji.LiveCDError("Failed to parse kickstart file "
                                   "'%s' : %s" % (kspath, e))

    def prepareKickstart(self, repo_info, target_info, arch, broot, opts):
        """
        Process the ks file to be used for controlled image generation. This
        method also uploads the modified kickstart file to the task output
        area.

        @args:
            target_info: a sesion.getBuildTarget() object
            repo_info: session.getRepo() object
            arch: canonical architecture name
            broot: a buildroot object
            kspath: absolute path to a kickstart file
        @returns:
            absolute path to a processed kickstart file within the buildroot
        """
        # Now we do some kickstart manipulation. If the user passed in a repo
        # url with --repo, then we substitute that in for the repo(s) specified
        # in the kickstart file. If --repo wasn't specified, then we use the
        # repo associated with the target passed in initially.
        repo_class = kscontrol.dataMap[self.ks.version]['RepoData']
        if not opts.get('ksrepo'):
            self.ks.handler.repo.repoList = []  # delete whatever the ks file told us
            if opts.get('repo'):
                user_repos = opts['repo']
                if isinstance(user_repos, six.string_types):
                    user_repos = user_repos.split(',')
                index = 0
                for user_repo in set(user_repos):
                    self.ks.handler.repo.repoList.append(repo_class(
                        baseurl=user_repo, name='koji-override-%i' % index))
                    index += 1
            else:
                path_info = koji.PathInfo(topdir=self.options.topurl)
                repopath = path_info.repo(repo_info['id'],
                                          target_info['build_tag_name'])
                baseurl = '%s/%s' % (repopath, arch)
                self.logger.debug('BASEURL: %s' % baseurl)
                self.ks.handler.repo.repoList.append(repo_class(
                    baseurl=baseurl, name='koji-%s-%i' % (target_info['build_tag_name'],
                                                          repo_info['id'])))
        # inject url if provided
        if opts.get('install_tree_url'):
            self.ks.handler.url(url=opts['install_tree_url'])

        # Write out the new ks file. Note that things may not be in the same
        # order and comments in the original ks file may be lost.
        kskoji = os.path.join(broot.tmpdir(), 'koji-image-%s-%i.ks' %
                              (target_info['build_tag_name'], self.id))
        koji.ensuredir(broot.tmpdir())
        with koji._open_text_file(kskoji, 'wt') as outfile:
            outfile.write(str(self.ks.handler))

        # put the new ksfile in the output directory
        if not os.path.exists(kskoji):
            raise koji.LiveCDError("KS file missing: %s" % kskoji)
        self.uploadFile(kskoji)
        return broot.path_without_to_within(kskoji)  # absolute path within chroot

    def getBootloaderAppend(self):
        """
        Return `bootloader --append`
        This is passed to livemedia `--extra-boot-args`
        """
        try:
            return self.ks.handler.bootloader.appendLine
        except AttributeError:
            return

    def getImagePackages(self, cachepath):
        """
        Read RPM header information from the yum cache available in the
        given path. Returns a list of dictionaries for each RPM included.
        """
        found = False
        hdrlist = []
        fields = ['name', 'version', 'release', 'epoch', 'arch',
                  'buildtime', 'sigmd5']
        for root, dirs, files in os.walk(cachepath):
            for f in files:
                if fnmatch(f, '*.rpm'):
                    pkgfile = os.path.join(root, f)
                    hdr = koji.get_header_fields(pkgfile, fields)
                    hdr['size'] = os.path.getsize(pkgfile)
                    hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
                    del hdr['sigmd5']
                    hdrlist.append(hdr)
                    found = True
        if not found:
            raise koji.LiveCDError('No repos found in yum cache!')
        return hdrlist

    def _shortenVolID(self, name, version, release):
        # Duplicated with pungi-fedora fedora.conf
        # see https://pagure.io/koji/pull-request/817
        substitutions = {
            'Beta': 'B',
            'Rawhide': 'rawh',
            'Astronomy_KDE': 'AstK',
            'Atomic': 'AH',
            'Cinnamon': 'Cinn',
            'Cloud': 'C',
            'Design_suite': 'Dsgn',
            'Electronic_Lab': 'Elec',
            'Everything': 'E',
                        'Games': 'Game',
                       'Images': 'img',
                      'Jam_KDE': 'Jam',
            'MATE_Compiz': 'MATE',
            # Note https://pagure.io/pungi-fedora/issue/533
            'Python-Classroom': 'Clss',
            'Python_Classroom': 'Clss',
                     'Robotics': 'Robo',
            'Scientific_KDE': 'SciK',
                     'Security': 'Sec',
                       'Server': 'S',
            'Workstation': 'WS',
            'WorkstationOstree': 'WS',
        }

        # Duplicated with pungi/util.py _apply_substitutions
        for k, v in sorted(to_list(substitutions.items()), key=lambda x: len(x[0]), reverse=True):
            if k in name:
                name = name.replace(k, v)
            if k in version:
                version = version.replace(k, v)
            if k in release:
                release = release.replace(k, v)

        volid = "%s-%s-%s" % (name, version, release)
        # Difference: pungi treats result more than 32 characters long as
        # fatal and raises an error
        return volid[:32]


# ApplianceTask begins with a mock chroot, and then installs appliance-tools
# into it via the appliance-build group. appliance-creator is then executed
# in the chroot to create the appliance image.
#
class ApplianceTask(ImageTask):

    Methods = ['createAppliance']
    _taskWeight = 1.5

    def getRootDevice(self):
        """
        Return the device name for the / partition, as specified in the
        kickstart file. Appliances should have this defined.
        """
        for part in self.ks.handler.partition.partitions:
            if part.mountpoint == '/':
                return part.disk
            elif part.fstype == 'btrfs':
                for s in self.ks.handler.btrfs.btrfsList:
                    if s.subvol and s.mountpoint == '/':
                        return part.disk
        raise koji.ApplianceError('kickstart lacks a "/" mountpoint')

    def handler(self, name, version, release, arch, target_info,
                build_tag, repo_info, ksfile, opts=None):

        if opts is None:
            opts = {}
        self.opts = opts
        broot = self.makeImgBuildRoot(build_tag, repo_info, arch,
                                      'appliance-build')
        kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name'])
        self.readKickstart(kspath, opts)
        kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts)
        # Figure out appliance-creator arguments, let it fail if something
        # is wrong.
        odir = 'app-output'
        opath = os.path.join(broot.tmpdir(), odir)
        # arbitrary paths in chroot
        cachedir = broot.tmpdir(within=True) + '/koji-appliance'
        app_log = broot.tmpdir(within=True) + '/appliance.log'
        os.mkdir(opath)

        cmd = ['/usr/bin/appliance-creator', '-c', kskoji, '-d', '-v',
               '--logfile', app_log, '--cache', cachedir, '-o', odir]
        for arg_name in ('vmem', 'vcpu', 'format'):
            arg = opts.get(arg_name)
            if arg is not None:
                cmd.extend(['--%s' % arg_name, arg])
        appname = '%s-%s-%s' % (name, version, release)
        cmd.extend(['--name', appname])
        cmd.extend(['--version', version, '--release', release])

        # Run appliance-creator
        rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
        self.uploadFile(os.path.join(broot.rootdir(), app_log[1:]))
        if rv:
            raise koji.ApplianceError(
                "Could not create appliance: %s" % parseStatus(rv, 'appliance-creator') +
                "; see root.log or appliance.log for more information")

        # Find the results
        results = []
        for directory, subdirs, files in os.walk(opath):
            for f in files:
                results.append(os.path.join(broot.tmpdir(),
                                            directory, f))
        self.logger.debug('output: %s' % results)
        if len(results) == 0:
            raise koji.ApplianceError("Could not find image build results!")
        logs = ['appliance.log', os.path.basename(ksfile), os.path.basename(kskoji)]
        logs.extend(broot.logs)
        imgdata = {
            'arch': arch,
            'rootdev': self.getRootDevice(),
            'task_id': self.id,
            'logs': logs,
            'name': name,
            'version': version,
            'release': release
        }
        imgdata['files'] = []
        for ofile in results:
            self.uploadFile(ofile)
            imgdata['files'].append(os.path.basename(ofile))

        # TODO: get file manifest from the appliance

        if not opts.get('scratch'):
            hdrlist = self.getImagePackages(os.path.join(broot.rootdir(),
                                                         cachedir[1:]))
            broot.markExternalRPMs(hdrlist)
            imgdata['rpmlist'] = hdrlist

        broot.expire()
        return imgdata

# LiveCDTask begins with a mock chroot, and then installs livecd-tools into it
# via the livecd-build group. livecd-creator is then executed in the chroot
# to create the LiveCD image.
#


class LiveCDTask(ImageTask):

    Methods = ['createLiveCD']
    _taskWeight = 1.5

    def genISOManifest(self, image, manifile):
        """
        Using iso9660 from pycdio, get the file manifest of the given image,
        and save it to the text file manifile.
        """
        fd = koji._open_text_file(manifile, 'wt')
        if not fd:
            raise koji.GenericError(
                'Unable to open manifest file (%s) for writing!' % manifile)
        iso = iso9660.ISO9660.IFS(source=image)
        if not iso.is_open():
            raise koji.GenericError(
                'Could not open %s as an ISO-9660 image!' % image)

        # image metadata
        id = iso.get_application_id()
        if id is not None:
            fd.write("Application ID: %s\n" % id)
        id = iso.get_preparer_id()
        if id is not None:
            fd.write("Preparer ID: %s\n" % id)
        id = iso.get_publisher_id()
        if id is not None:
            fd.write("Publisher ID: %s\n" % id)
        id = iso.get_system_id()
        if id is not None:
            fd.write("System ID: %s\n" % id)
        id = iso.get_volume_id()
        if id is not None:
            fd.write("Volume ID: %s\n" % id)
        id = iso.get_volumeset_id()
        if id is not None:
            fd.write("Volumeset ID: %s\n" % id)

        fd.write('\nSize(bytes)  File Name\n')
        manifest = self.listISODir(iso, '/')
        for a_file in manifest:
            fd.write(a_file)
        fd.close()
        iso.close()

    def listISODir(self, iso, path):
        """
        Helper function called recursively by genISOManifest. Returns a
        listing of files/directories at the given path in an iso image obj.
        """
        manifest = []
        file_stats = iso.readdir(path)
        for stat in file_stats:
            filename = stat[0]
            size = stat[2]
            is_dir = stat[4] == 2

            if filename == '..':
                continue
            elif filename == '.':
                # path should always end in a trailing /
                filepath = path
            else:
                filepath = path + filename
                # identify directories with a trailing /
                if is_dir:
                    filepath += '/'

            if is_dir and filename != '.':
                # recurse into subdirectories
                manifest.extend(self.listISODir(iso, filepath))
            else:
                # output information for the current directory and files
                manifest.append("%-10d   %s\n" % (size, filepath))

        return manifest

    def handler(self, name, version, release, arch, target_info,
                build_tag, repo_info, ksfile, opts=None):

        if opts is None:
            opts = {}
        self.opts = opts

        broot = self.makeImgBuildRoot(build_tag, repo_info, arch,
                                      'livecd-build')
        kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name'])
        self.readKickstart(kspath, opts)
        kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts)

        # arbitrary paths in chroot
        cachedir = broot.tmpdir(within=True) + '/koji-livecd'
        livecd_log = broot.tmpdir(within=True) + '/livecd.log'
        cmd = ['/usr/bin/livecd-creator', '-c', kskoji, '-d', '-v',
               '--logfile', livecd_log, '--cache', cachedir]
        isoname = '%s-%s-%s' % (name, version, release)
        volid = opts.get('volid')
        if not volid:
            volid = self._shortenVolID(name, version, release)
        if len(volid) > 32:
            raise koji.LiveCDError('volume ID is longer than 32 characters')
        cmd.extend(['-f', volid])

        # Run livecd-creator
        rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)
        self.uploadFile(os.path.join(broot.rootdir(), livecd_log[1:]))
        if rv:
            raise koji.LiveCDError(
                'Could not create LiveCD: %s' % parseStatus(rv, 'livecd-creator') +
                '; see root.log or livecd.log for more information')

        # Find the resultant iso
        # The cwd of the livecd-creator process is tmpdir() in the chroot, so
        # that is where it writes the .iso
        files = os.listdir(broot.tmpdir())
        isofile = None
        for afile in files:
            if afile.endswith('.iso'):
                if not isofile:
                    isofile = afile
                else:
                    raise koji.LiveCDError(
                        'multiple .iso files found: %s and %s' % (isofile, afile))
        if not isofile:
            raise koji.LiveCDError('could not find iso file in chroot')
        isosrc = os.path.join(broot.tmpdir(), isofile)

        # copy the iso out of the chroot. If we were given an isoname,
        # this is where the renaming happens.
        self.logger.debug('uploading image: %s' % isosrc)
        isoname += '.iso'

        # Generate the file manifest of the image, upload the results
        manifest = os.path.join(broot.resultdir(), 'manifest.log')
        self.genISOManifest(isosrc, manifest)
        self.uploadFile(manifest)
        self.uploadFile(isosrc, remoteName=isoname)

        logs = ['livecd.log', os.path.basename(ksfile), os.path.basename(kskoji)]
        logs.extend(broot.logs)
        imgdata = {'arch': arch,
                   'files': [isoname],
                   'rootdev': None,
                   'task_id': self.id,
                   'logs': logs,
                   'name': name,
                   'version': version,
                   'release': release
                   }
        if not opts.get('scratch'):
            hdrlist = self.getImagePackages(os.path.join(broot.rootdir(),
                                                         cachedir[1:]))
            imgdata['rpmlist'] = hdrlist
            broot.markExternalRPMs(hdrlist)

        broot.expire()
        return imgdata


# livemedia-creator
class LiveMediaTask(ImageTask):

    Methods = ['createLiveMedia']
    _taskWeight = 1.5
    # For livemedia-creator we do not want to bind mount /dev, see
    # https://bugzilla.redhat.com/show_bug.cgi?id=1315541
    bind_opts = {}

    def fetch_lorax_templates_from_scm(self, build_root):
        """
        Checkout the lorax templates from SCM for use by livemedia-creator.

        This will make a checkout of the lorax templates from SCM so that they
        may be passed to livemedia-creator.  Here we are operating outside the
        chroot of the BuildRoot.  The following options are essential:
            - lorax_url points to the SCM containing the templates.
            - lorax_dir provides a relative reference to the templates within
            the checkout.

        :param build_root:
            The BuildRoot instance to receive the checkout.
        :return:
            An absolute path (from within the chroot) to where livemedia-creator
            can find the checked out templates.
        """
        scm = SCM(self.opts['lorax_url'], allow_password=self.options.allow_password_in_scm_url)
        scm.assert_allowed(allowed=self.options.allowed_scms,
                           session=self.session,
                           by_config=self.options.allowed_scms_use_config,
                           by_policy=self.options.allowed_scms_use_policy,
                           policy_data={
                               'user_id': self.taskinfo['owner'],
                               'channel': self.session.getChannel(self.taskinfo['channel_id'],
                                                                  strict=True)['name'],
                               'scratch': self.opts.get('scratch')
                           })
        logfile = os.path.join(self.workdir, 'lorax-templates-checkout.log')
        checkout_dir = scm.checkout(build_root.tmpdir(),
                                    self.session, self.getUploadDir(), logfile)
        return os.path.join(build_root.path_without_to_within(checkout_dir),
                            self.opts['lorax_dir'])

    def genISOManifest(self, image, manifile):
        """
        Using iso9660 from pycdio, get the file manifest of the given image,
        and save it to the text file manifile.
        """
        fd = koji._open_text_file(manifile, 'wt')
        if not fd:
            raise koji.GenericError(
                'Unable to open manifest file (%s) for writing!' % manifile)
        iso = iso9660.ISO9660.IFS(source=image)
        if not iso.is_open():
            raise koji.GenericError(
                'Could not open %s as an ISO-9660 image!' % image)

        # image metadata
        id = iso.get_application_id()
        if id is not None:
            fd.write("Application ID: %s\n" % id)
        id = iso.get_preparer_id()
        if id is not None:
            fd.write("Preparer ID: %s\n" % id)
        id = iso.get_publisher_id()
        if id is not None:
            fd.write("Publisher ID: %s\n" % id)
        id = iso.get_system_id()
        if id is not None:
            fd.write("System ID: %s\n" % id)
        id = iso.get_volume_id()
        if id is not None:
            fd.write("Volume ID: %s\n" % id)
        id = iso.get_volumeset_id()
        if id is not None:
            fd.write("Volumeset ID: %s\n" % id)

        fd.write('\nSize(bytes)  File Name\n')
        manifest = self.listISODir(iso, '/')
        for a_file in manifest:
            fd.write(a_file)
        fd.close()
        iso.close()

    def listISODir(self, iso, path):
        """
        Helper function called recursively by genISOManifest. Returns a
        listing of files/directories at the given path in an iso image obj.
        """
        manifest = []
        file_stats = iso.readdir(path)
        for stat in file_stats:
            filename = stat[0]
            size = stat[2]
            is_dir = stat[4] == 2

            if filename == '..':
                continue
            elif filename == '.':
                # path should always end in a trailing /
                filepath = path
            else:
                filepath = path + filename
                # identify directories with a trailing /
                if is_dir:
                    filepath += '/'

            if is_dir and filename != '.':
                # recurse into subdirectories
                manifest.extend(self.listISODir(iso, filepath))
            else:
                # output information for the current directory and files
                manifest.append("%-10d   %s\n" % (size, filepath))

        return manifest

    def handler(self, name, version, release, arch, target_info,
                build_tag, repo_info, ksfile, opts=None):

        if opts is None:
            opts = {}
        self.opts = opts

        broot = self.makeImgBuildRoot(build_tag, repo_info, arch,
                                      'livemedia-build')
        kspath = self.fetchKickstart(broot, ksfile, target_info['build_tag_name'])
        self.readKickstart(kspath, opts)
        kskoji = self.prepareKickstart(repo_info, target_info, arch, broot, opts)
        b_append = self.getBootloaderAppend()

        # arbitrary paths in chroot
        livemedia_log = broot.tmpdir(within=True) + '/lmc-logs/livemedia-out.log'
        resultdir = broot.tmpdir(within=True) + '/lmc'

        # Common LMC command setup, needs extending
        cmd = ['/sbin/livemedia-creator',
               '--ks', kskoji,
               '--logfile', livemedia_log,
               '--no-virt',
               '--resultdir', resultdir,
               '--project', name,
               # '--tmp', '/tmp'
               ]

        volid = opts.get('volid')
        if not volid:
            volid = self._shortenVolID(name, version, release)
        if len(volid) > 32:
            raise koji.LiveMediaError('volume ID is longer than 32 characters')

        # note: at the moment, we are only generating live isos. We may add support
        # for other types in the future

        cmd.extend(['--make-iso',
                    '--volid', volid,
                    '--iso-only',
                    ])

        isoname = '%s-%s-%s-%s.iso' % (name, arch, version, release)
        cmd.extend(['--iso-name', isoname,
                    '--releasever', version,
                    ])

        if arch == 'x86_64' and not opts.get('nomacboot'):
            cmd.append('--macboot')
        else:
            cmd.append('--nomacboot')

        if b_append:
            cmd.extend(['--extra-boot-args', '\"%s\"' % b_append])

        if 'lorax_url' in self.opts:
            templates_dir = self.fetch_lorax_templates_from_scm(broot)
            cmd.extend(['--lorax-templates', templates_dir])

        if self.opts.get('squashfs_only'):
            cmd.append('--squashfs-only')

        if isinstance(self.opts.get('compress_arg'), (list, tuple)):
            for com_arg in self.opts['compress_arg']:
                cmd.extend(['--compress-arg', com_arg])
        # Run livemedia-creator
        rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd)

        # upload logs
        logdirs = [
            os.path.join(broot.tmpdir(), 'lmc-logs'),
            os.path.join(broot.tmpdir(), 'lmc-logs/anaconda'),
        ]
        for logdir in logdirs:
            if not os.path.isdir(logdir):
                continue
            for filename in os.listdir(logdir):
                if not filename.endswith('.log'):
                    continue
                filepath = os.path.join(logdir, filename)
                if os.stat(filepath).st_size == 0:
                    continue
                # avoid file duplication between directories by prefixing anaconda logs
                if logdir.endswith('anaconda'):
                    self.uploadFile(os.path.join(filepath), remoteName='anaconda-%s' % filename)
                    continue

                self.uploadFile(os.path.join(filepath))

        if rv:
            raise koji.LiveMediaError(
                'Could not create LiveMedia: %s' % parseStatus(rv, 'livemedia-creator') +
                '; see root.log or livemedia-out.log for more information')

        # Find the resultant iso
        # The cwd of the livemedia-creator process is broot.tmpdir() in the chroot, so
        # that is where it writes the .iso
        rootresultsdir = os.path.join(broot.rootdir(), resultdir.lstrip('/'))
        files = os.listdir(rootresultsdir)
        isofile = None
        for afile in files:
            if afile.endswith('.iso'):
                if not isofile:
                    isofile = afile
                else:
                    raise koji.LiveMediaError(
                        'multiple .iso files found: %s and %s' % (isofile, afile))
        if not isofile:
            raise koji.LiveMediaError('could not find iso file in chroot')
        isosrc = os.path.join(rootresultsdir, isofile)

        # Generate the file manifest of the image, upload the results
        manifest = os.path.join(broot.resultdir(), 'manifest.log')
        self.genISOManifest(isosrc, manifest)
        self.uploadFile(manifest)
        self.logger.debug('uploading image: %s' % isosrc)
        self.uploadFile(isosrc, remoteName=isoname)

        logs = ['livemedia-out.log', os.path.basename(ksfile), os.path.basename(kskoji)]
        logs.extend(broot.logs)
        imgdata = {'arch': arch,
                   'files': [isoname],
                   'rootdev': None,
                   'task_id': self.id,
                   'logs': logs,
                   'name': name,
                   'version': version,
                   'release': release
                   }
        if not opts.get('scratch'):
            # TODO - generate list of rpms in image
            # (getImagePackages doesn't work here)
            # hdrlist = self.getImagePackages(os.path.join(broot.rootdir(),
            #                                cachedir[1:]))
            imgdata['rpmlist'] = []
            # broot.markExternalRPMs(hdrlist)

        broot.expire()
        return imgdata

# A generic task for building disk images using Oz
# Other Oz-based image handlers should inherit this.


class OzImageTask(BaseTaskHandler):
    Methods = []
    supported_formats = {}

    def fetchKickstart(self, build_tag):
        """
        Retrieve the kickstart file we were given (locally or remotely) and
        upload it to the hub.

        Note that if the KS file existed locally, then "ksfile" is a relative
        path to it in the /mnt/koji/work directory. If not, then it is still
        the parameter the user passed in initially, and we assume it is a
        relative path in a remote scm. The user should have passed in an scm
        url with --ksurl.

        @args: build_tag: build tag name
               use self.opts for options
        @returns:
            absolute path to the retrieved kickstart file
        """
        ksfile = self.opts.get('kickstart')
        self.logger.debug("ksfile = %s" % ksfile)
        if self.opts.get('ksurl'):
            scm = SCM(self.opts['ksurl'], allow_password=self.options.allow_password_in_scm_url)
            scm.assert_allowed(allowed=self.options.allowed_scms,
                               session=self.session,
                               by_config=self.options.allowed_scms_use_config,
                               by_policy=self.options.allowed_scms_use_policy,
                               policy_data={
                                   'user_id': self.taskinfo['owner'],
                                   'channel': self.session.getChannel(self.taskinfo['channel_id'],
                                                                      strict=True)['name'],
                                   'scratch': self.opts.get('scratch')
                               })
            logfile = os.path.join(self.workdir, 'checkout-%s.log' % self.arch)
            self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
                               build_tag=build_tag, scratch=self.opts.get('scratch'))
            scmsrcdir = scm.checkout(self.workdir, self.session,
                                     self.getUploadDir(), logfile)
            self.run_callbacks("postSCMCheckout",
                               scminfo=scm.get_info(),
                               build_tag=build_tag,
                               scratch=self.opts.get('scratch'),
                               srcdir=scmsrcdir)
            kspath = os.path.join(scmsrcdir, os.path.basename(ksfile))
        else:
            tops = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')])
            tops['tempdir'] = self.workdir
            with koji.openRemoteFile(ksfile, **tops) as ks_src:
                kspath = os.path.join(self.workdir, os.path.basename(ksfile))
                with open(kspath, 'wb') as ks_dest:
                    ks_dest.write(ks_src.read())

        if not os.path.exists(kspath):
            raise koji.BuildError('Kickstart %s does not exist.' % kspath)

        self.logger.debug('uploading kickstart from here: %s' % kspath)
        self.uploadFile(kspath)  # upload the original ks file
        return kspath  # absolute path to the ks file

    def readKickstart(self, kspath):
        """
        Read a kickstart file and save the ks object as a task member.

        @args:
            kspath: path to a kickstart file
        @returns:
            a kickstart object returned by pykickstart
        """
        # XXX: If the ks file came from a local path and has %include
        #      macros, Oz will fail because it can only handle flat files.
        #      We require users to flatten their kickstart file.
        if self.opts.get('ksversion'):
            version = ksparser.version.makeVersion(self.opts['ksversion'])
        else:
            version = ksparser.version.makeVersion()
        ks = ksparser.KickstartParser(version)
        self.logger.debug('attempting to read kickstart: %s' % kspath)
        try:
            ks.readKickstart(kspath)
        except IOError as e:
            raise koji.BuildError("Failed to read kickstart file "
                                  "'%s' : %s" % (kspath, e))
        except kserrors.KickstartError as e:
            raise koji.BuildError("Failed to parse kickstart file "
                                  "'%s' : %s" % (kspath, e))
        return ks

    def prepareKickstart(self, kspath, install_tree):
        """
        Process the ks file to be used for controlled image generation. This
        method also uploads the modified kickstart file to the task output
        area on the hub.

        @args:
            kspath: a path to a kickstart file
        @returns:
            a kickstart object with koji-specific modifications
        """
        ks = self.readKickstart(kspath)
        # Now we do some kickstart manipulation. If the user passed in a repo
        # url with --repo, then we substitute that in for the repo(s) specified
        # in the kickstart file. If --repo wasn't specified, then we use the
        # repo associated with the target passed in initially.
        ks.handler.repo.repoList = []  # delete whatever the ks file told us
        repo_class = kscontrol.dataMap[ks.version]['RepoData']
        # only use noverifyssl if allowed in kojid.conf
        if self.opts.get('noverifyssl') and not self.options.allow_noverifyssl:
            raise koji.BuildError("noverifyssl option is not enabled")
        noverifyssl = self.options.allow_noverifyssl and self.opts.get('noverifyssl')
        # TODO: sensibly use "url" and "repo" commands in kickstart
        if self.opts.get('repo'):
            # the user used --repo at least once
            user_repos = self.opts.get('repo')
            index = 0
            for user_repo in set(user_repos):
                repo_url = user_repo.replace('$arch', self.arch)
                ks.handler.repo.repoList.append(repo_class(
                    baseurl=repo_url, name='koji-override-%i' % index,
                    noverifyssl=noverifyssl))
                index += 1
        else:
            # --repo was not given, so we use the target's build repo
            path_info = koji.PathInfo(topdir=self.options.topurl)
            repopath = path_info.repo(self.repo_info['id'],
                                      self.target_info['build_tag_name'])
            baseurl = '%s/%s' % (repopath, self.arch)
            self.logger.debug('BASEURL: %s' % baseurl)
            ks.handler.repo.repoList.append(repo_class(
                baseurl=baseurl, name='koji-override-0',
                noverifyssl=noverifyssl))
        # inject the URL of the install tree into the kickstart
        ks.handler.url(url=install_tree, noverifyssl=noverifyssl)
        return ks

    def writeKickstart(self, ksobj, ksname):
        """
        Write out the new ks file. Note that things may not be in the same
        order and comments in the original ks file may be lost.

        @args:
            ksobj: a pykickstart object of what we want to write
            ksname: file name for the kickstart
        @returns:
            an absolute path to the kickstart file we wrote
        """
        kspath = os.path.join(self.workdir, ksname)
        with koji._open_text_file(kspath, 'wt') as outfile:
            outfile.write(str(ksobj.handler))

        # put the new ksfile in the output directory
        if not os.path.exists(kspath):
            raise koji.BuildError("KS file missing: %s" % kspath)
        self.uploadFile(kspath)  # upload the modified ks file
        return kspath

    def makeConfig(self):
        """
        Generate a configuration dict for ImageFactory. This will override
        anything in the /etc config files. We do this forcibly so that it is
        impossible for Koji to use any image caches or leftover metadata from
        other images created by the service.

        @args: none
        @returns:
            a dictionary used for configuring ImageFactory to built an image
            the way we want
        """
        return {
            # Oz specific
            'oz_data_dir': os.path.join(self.workdir, 'oz_data'),
            'oz_screenshot_dir': os.path.join(self.workdir, 'oz_screenshots'),
            # IF specific
            'imgdir': os.path.join(self.workdir, 'scratch_images'),
            'tmpdir': os.path.join(self.workdir, 'oz-tmp'),
            'verbose': True,
            'timeout': self.options.oz_install_timeout or None,
            'output': 'log',
            'raw': False,
            'debug': True,
            'image_manager': 'file',
            'plugins': '/etc/imagefactory/plugins.d',
            'rhevm_image_format': 'qcow2',
            'tdl_require_root_pw': False,
            'image_manager_args': {
                'storage_path': os.path.join(self.workdir, 'output_image')},
        }

    def makeTemplate(self, name, inst_tree):
        """
        Generate a simple "TDL" for ImageFactory to build an image with.

        @args:
            name: a name for the image
            inst_tree: a string, a URL to the install tree (a compose)
        @returns:
            An XML string that imagefactory can consume
        """
        # we have to split this up so the variable substitution works
        # XXX: using a packages section (which we don't) will have IF boot the
        #      image and attempt to ssh in. This breaks docker image creation.
        # TODO: intelligently guess the distro based on the install tree URL
        distname, distver = self.parseDistro(self.opts.get('distro'))
        if self.arch in ['armhfp', 'armv7hnl', 'armv7hl']:
            arch = 'armv7l'
        else:
            arch = self.arch
        template = """<template>
    <name>%s</name>
        <os>
            <name>%s</name>
            <version>%s</version>
            <arch>%s</arch>
            <install type='url'>
                <url>%s</url>
            </install>
        """ % (name, distname, distver, arch, inst_tree)
        template += ("<icicle>\n"
                     "              <extra_command>rpm -qa --qf"
                     " '%{NAME},%{VERSION},%{RELEASE},%{ARCH},%{EPOCH},%{SIZE},%{SIGMD5},"
                     "%{BUILDTIME}\\n'</extra_command>\n"
                     "          </icicle>\n"
                     "      ")
        # TODO: intelligently guess the size based on the kickstart file
        template += """</os>
    <description>%s OS</description>
    <disk>
        <size>%s</size>
    </disk>
</template>
""" % (name, self.opts.get('disk_size'))  # noqa: E501
        return template

    def parseDistro(self, distro):
        """
        Figure out the distribution name and version we are going to build an
        image on.

        args:
            a string of the form: RHEL-X.Y, Fedora-NN, CentOS-X.Y, or SL-X.Y
        returns:
            a 2-element list, depends on the distro where the split happened
        """
        if distro.startswith('RHEL'):
            major, minor = distro.split('.')
            if major == 'RHEL-5':
                minor = 'U' + minor
            return major, minor
        elif distro.startswith('Fedora'):
            return distro.split('-')
        elif distro.startswith('CentOS'):
            return distro.split('.')
        elif distro.startswith('SL'):
            return distro.split('.')
        else:
            raise koji.BuildError('Unknown or supported distro given: %s' % distro)

    def fixImageXML(self, format, filename, xmltext):
        """
        The XML generated by Oz/ImageFactory knows nothing about the name
        or image format conversions Koji does. We fix those values in the
        libvirt XML and write the changes out to a file, the path of which
        we return.

        @args:
            format = raw, qcow2, vmdk, etc... a string representation
            filename = the name of the XML file we will save this too
            xmltext = the libvirt XML to start with
        @return:
            an absolute path to the modified XML
        """
        newxml = xml.dom.minidom.parseString(xmltext)  # nosec
        ename = newxml.getElementsByTagName('name')[0]
        ename.firstChild.nodeValue = self.imgname
        esources = newxml.getElementsByTagName('source')
        for e in esources:
            if e.hasAttribute('file'):
                e.setAttribute('file', '%s.%s' % (self.imgname, format))
        edriver = newxml.getElementsByTagName('driver')[0]
        edriver.setAttribute('type', format)
        if not self.supported_formats.get(format, {}).get('qemu'):
            edriver.setAttribute('type-warning',
                                 "%s is not qemu-supported format, "
                                 "you need to convert image before use "
                                 "and update driver+source accordingly." % format)
        xml_path = os.path.join(self.workdir, filename)
        with koji._open_text_file(xml_path, 'wt') as xmlfd:
            xmlfd.write(newxml.toprettyxml())
        return xml_path

    def getScreenshot(self):
        """
        Locate a screenshot taken by libvirt in the case of build failure,
        if it exists. If it does, return the path, else return None.

        @args: none
        @returns: a path to a screenshot take by libvirt
        """
        shotdir = os.path.join(self.workdir, 'oz_screenshots')
        screenshot = None
        found = glob.glob(os.path.join(shotdir, '*.ppm'))
        if len(found) > 0:
            screenshot = found[0]
        found = glob.glob(os.path.join(shotdir, '*.png'))
        if len(found) > 0:
            screenshot = found[0]
        return screenshot


class BaseImageTask(OzImageTask):

    Methods = ['createImage']
    _taskWeight = 2.0

    def __init__(self, *args, **kwargs):
        super(BaseImageTask, self).__init__(*args, **kwargs)
        '''
        format: {
            'qemu': bool -supported format by qemu
            'fcall': function to handle creation of this format
        }
        '''
        self.supported_formats = {
            'docker': {'qemu': False, 'fcall': self._buildDocker},
            'liveimg-squashfs': {'qemu': False, 'fcall': self._buildSquashfs},
            'qcow': {'qemu': True, 'fcall': self._buildConvert},
            'qcow2': {'qemu': True, 'fcall': self._buildConvert},
            'raw': {'qemu': True, 'fcall': self._buildBase},
            'raw-xz': {'qemu': False, 'fcall': self._buildXZ},
            'rhevm-ova': {'qemu': False, 'fcall': self._buildOVA},
            'tar-gz': {'qemu': False, 'fcall': self._buildTarGZ},
            'vagrant-hyperv': {'qemu': False, 'fcall': self._buildOVA},
            'vagrant-libvirt': {'qemu': False, 'fcall': self._buildOVA},
            'vagrant-virtualbox': {'qemu': False, 'fcall': self._buildOVA},
            'vagrant-vmware-fusion': {'qemu': False, 'fcall': self._buildOVA},
            'vdi': {'qemu': True, 'fcall': self._buildConvert},
            'vmdk': {'qemu': True, 'fcall': self._buildConvert},
            'vpc': {'qemu': True, 'fcall': self._buildConvert},
            'vsphere-ova': {'qemu': False, 'fcall': self._buildOVA},
        }

    def _format_deps(self, formats):
        """
        Return a dictionary where the keys are the image formats we need to
        build/convert, and the values are booleans that indicate whether the
        output should be included in the task results.

        Some image formats require others to be processed first, which is why
        we have to do this. raw files in particular may not be kept.
        """
        for f in formats:
            if f not in self.supported_formats.keys():
                raise koji.ApplianceError('Invalid format: %s' % f)
        f_dict = dict((f, True) for f in formats)

        # If a user requests 1 or more image formats (with --format) we do not
        # by default include the raw disk image in the results, because it is
        # 10G in size. To override this behavior, the user must specify
        # "--format raw" in their command. If --format was not used at all,
        # then we do include the raw disk image by itself.
        if len(formats) == 0:
            # we only want a raw disk image (no format option given)
            f_dict['raw'] = True
        elif 'raw' not in f_dict:
            f_dict['raw'] = False
        self.logger.debug('Image delivery plan: %s' % f_dict)
        return f_dict

    def do_images(self, ks, template, inst_tree):
        """
        Call out to ImageFactory to build the image(s) we want. Returns a dict
        of details for each image type we had to ask ImageFactory to build
        """
        # add a handler to the logger so that we capture ImageFactory's logging
        self.fhandler = logging.FileHandler(self.ozlog)
        self.bd = BuildDispatcher()
        self.tlog = logging.getLogger()
        self.tlog.setLevel(logging.DEBUG)
        self.tlog.addHandler(self.fhandler)
        images = {}
        random.seed()  # necessary to ensure a unique mac address
        params = {'install_script': str(ks.handler),
                  'offline_icicle': True}
        # build the base (raw) image
        self.base_img = self._buildBase(template, params)
        images['raw'] = {'image': self.base_img.base_image.data,
                         'icicle': self.base_img.base_image.icicle}
        # Do the rest of the image types (everything but raw)
        for format in self.formats:
            if format == 'raw':
                continue
            self.logger.info('dispatching %s image builder' % format)
            images[format] = self.supported_formats[format]['fcall'](format)
        imginfo = self._processXML(images)
        self.tlog.removeHandler(self.fhandler)
        self.uploadFile(self.ozlog)
        return imginfo

    def _processXML(self, images):
        """
        Produce XML that libvirt can import to create a domain based on image(s)
        we produced. We save the location of the XML file in the dictionary
        it corresponds to here.

        @args:
            images - a dict where the keys are image formats, and the values
                are dicts with details about the image (location, icicle, etc)
        @returns:
            a dictionary just like "images" but with a new key called "libvirt"
            that points to the path of the XML file for that image
        """
        imginfo = {}
        for fmt in images:
            imginfo[fmt] = images[fmt]
            lxml = self.fixImageXML(fmt, 'libvirt-%s-%s.xml' % (fmt, self.arch),
                                    self.base_img.base_image.parameters['libvirt_xml'])
            imginfo[fmt]['libvirt'] = lxml
        return imginfo

    def _checkImageState(self, image):
        """
        Query ImageFactory for details of a dispatched image build. If it is
        FAILED we raise an exception.

        @args:
            image - a build dispatcher object returned by a BuildDispatcher
        @returns: nothing
        """
        if image.target_image:
            status = image.target_image.status
            details = image.target_image.status_detail['error']
        else:
            status = image.base_image.status
            details = image.base_image.status_detail['error']
        self.logger.debug('check image results: %s' % status)
        if status == 'FAILED':
            scrnshot = self.getScreenshot()
            if scrnshot:
                ext = scrnshot[-3:]
                self.uploadFile(scrnshot, remoteName='screenshot.%s' % ext)
            if image.os_plugin:
                image.os_plugin.abort()  # forcibly tear down the VM
            # TODO abort when a task is CANCELLED
            if not self.session.checkUpload('', os.path.basename(self.ozlog)):
                self.tlog.removeHandler(self.fhandler)
                self.uploadFile(self.ozlog)
            if 'No disk activity' in details:
                details = 'Automated install failed or prompted for input. ' \
                          'See the screenshot in the task results for more information'
            raise koji.ApplianceError('Image status is %s: %s' %
                                      (status, details))

    def _mergeFactoryParams(self, img_opts, fixed_params):
        """
        Merge any KV pairs passed in via --factory-parameter CLI options
        into an existing dictionary that will eventually be passed to
        factory build commands.  This allows a fairly generic mechanism
        for parameter passthrough to plugins and the base builder that does
        not require patching the builder and CLI each time.

        @args:
            img_opts - an existing dict with any pre-existing parameters
            fixed_params - list of parameters that must not be overridden
        @returns:
            nothing - dict is modified in place
        """
        if self.opts.get('factory_parameter'):
            for kvp in self.opts.get('factory_parameter'):
                if kvp[0] not in fixed_params:
                    img_opts[kvp[0]] = kvp[1]

    def _buildBase(self, template, params, wait=True):
        """
        Build a base image using ImageFactory. This is a "raw" image.

        @args:
            template - an XML string for the TDL
            params - a dict that controls some ImageFactory settings
            wait - call join() on the building thread if True
        @returns:
            a dict with some metadata about the image (includes an icicle)
        """
        # TODO: test the failure case where IF itself throws an exception
        #       ungracefully (missing a plugin for example)
        #       may need to still upload ozlog and remove the log handler
        self.logger.info('dispatching a baseimg builder')
        self.logger.debug('templates: %s' % template)
        self.logger.debug('pre-merge params: %s' % params)
        # We enforce various things related to the ks file - do not allow override
        self._mergeFactoryParams(params, ['install_script'])
        self.logger.debug('post-merge params: %s' % params)
        base = self.bd.builder_for_base_image(template, parameters=params)
        if wait:
            base.base_thread.join()
            self._checkImageState(base)
        return base

    def _buildXZ(self, format):
        """
        Use xz to compress a raw disk image. This is very straightforward.

        @args:
            format - a string representing the image format, "raw-xz"
        @returns:
            a dict with some metadata about the image
        """
        newimg = os.path.join(self.workdir, self.imgname + '.raw.xz')
        rawimg = os.path.join(self.workdir, self.imgname + '.raw')
        cmd = ['/bin/cp', self.base_img.base_image.data, rawimg]
        conlog = os.path.join(self.workdir,
                              'xz-cp-%s-%s.log' % (format, self.arch))
        log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
                   logerror=1)
        cmd = ['/usr/bin/xz', self.options.xz_options, rawimg]
        conlog = os.path.join(self.workdir,
                              'xz-%s-%s.log' % (format, self.arch))
        log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
                   logerror=1)
        return {'image': newimg}

    def _buildTarGZ(self, format):
        """
        Use tar and gzip to compress a raw disk image.

        @args:
            format - not used, we only handle tar-gz
        @returns:
            a dict with some metadata about the image
        """
        orig = self.base_img.base_image.data
        newimg = os.path.join(self.workdir, self.imgname + '.tar.gz')

        # see also: https://cloud.google.com/compute/docs/creating-custom-image

        # the image in the tarball must be named disk.raw
        imgdir = os.path.dirname(orig)
        rawimg = os.path.join(imgdir, 'disk.raw')
        os.link(orig, rawimg)

        # make the tarball
        cmd = ['/bin/tar', '-Sczvf', newimg, 'disk.raw']
        conlog = os.path.join(self.workdir, 'tar-gz-%s.log' % self.arch)
        log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
                   logerror=1, cwd=imgdir)

        # now that we've made the tarball, we don't need this hardlink
        os.unlink(rawimg)

        return {'image': newimg}

    def _buildSquashfs(self, format):
        """
        Use squashfs to wrap a raw disk image into liveimg compatible image.
        This can be used by dracut for booting or anaconda to install.

        @args:
            format - a string representing the image format, "liveimg-squashfs"
        @returns:
            a dict with some metadata about the image
        """
        newimg = os.path.join(self.workdir, self.imgname + '.squashfs')
        fsimg = os.path.join(self.workdir, 'squashfs-root/LiveOS/rootfs.img')

        os.makedirs(os.path.join(self.workdir, 'squashfs-root/LiveOS'))

        cmd = ['/bin/dd', 'conv=sparse', 'bs=1M',
               'skip=1',  # FIXME Hack to strip the disklabel
               'if=%s' % self.base_img.base_image.data,
               'of=%s' % fsimg]
        conlog = os.path.join(self.workdir,
                              'squashfs-dd-%s-%s.log' % (format, self.arch))
        log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
                   logerror=1)
        cmd = ['/usr/sbin/mksquashfs', os.path.join(self.workdir, 'squashfs-root'),
               newimg, '-comp', 'xz', '-noappend']
        conlog = os.path.join(self.workdir,
                              'squashfs-mksquashfs-%s-%s.log' % (format, self.arch))
        log_output(self.session, cmd[0], cmd, conlog, self.getUploadDir(),
                   logerror=1)
        return {'image': newimg}

    def _buildOVA(self, format):
        """
        Build an OVA target image. This is a format supported by RHEV and
        vSphere

        @args:
            format - a string representing the image format, "rhevm-ova"
        @returns:
            a dict with some metadata about the image
        """
        img_opts = {}
        if self.opts.get('ova_option'):
            img_opts = dict([o.split('=') for o in self.opts.get('ova_option')])
        # As far as Image Factory is concerned, vagrant boxes are just another type of OVA
        # We communicate the desire for vagrant-specific formatting by adding the *_ova_format
        # options and turning the underlying format option back into one of the two target
        # image types ('vsphere-ova' or 'rhevm-ova') that are used to generate the intermediate
        # disk image
        fixed_params = []
        if format == 'vagrant-virtualbox':
            format = 'vsphere-ova'
            img_opts['vsphere_ova_format'] = 'vagrant-virtualbox'
            fixed_params = ['vsphere_ova_format']
        if format == 'vagrant-libvirt':
            format = 'rhevm-ova'
            img_opts['rhevm_ova_format'] = 'vagrant-libvirt'
            fixed_params = ['rhevm_ova_format']
        if format == 'vagrant-vmware-fusion':
            format = 'vsphere-ova'
            img_opts['vsphere_ova_format'] = 'vagrant-vmware-fusion'
            # The initial disk image transform for VMWare Fusion/Workstation requires a "standard"
            # VMDK, not the stream oriented format used for VirtualBox or regular VMWare OVAs
            img_opts['vsphere_vmdk_format'] = 'standard'
            fixed_params = ['vsphere_ova_format', 'vsphere_vmdk_format']
        if format == 'vagrant-hyperv':
            format = 'hyperv-ova'
            img_opts['hyperv_ova_format'] = 'hyperv-vagrant'
            fixed_params = ['hyperv_ova_format']
        targ = self._do_target_image(self.base_img.base_image.identifier,
                                     format.replace('-ova', ''), img_opts=img_opts,
                                     fixed_params=fixed_params)
        targ2 = self._do_target_image(targ.target_image.identifier, 'OVA',
                                      img_opts=img_opts, fixed_params=fixed_params)
        return {'image': targ2.target_image.data}

    def _buildDocker(self, format):
        """
        Build a base docker image. This image will be tagged with the NVR.A
        automatically because we name it that way in the ImageFactory TDL.

        @args:
            format - the string "docker"
        @returns:
            a dict with some metadata about the image
        """
        img_opts = {'compress': 'xz'}
        targ = self._do_target_image(self.base_img.base_image.identifier,
                                     'docker', img_opts=img_opts)
        return {'image': targ.target_image.data}

    def _do_target_image(self, base_id, image_type, img_opts=None, fixed_params=None):
        """
        A generic method for building what ImageFactory calls "target images".
        These are images based on a raw disk that was built before using the
        _buildBase method.

        @args:
            base_id - a string ID of the image to build off of
            image_type - a string representing the target type. ImageFactory
                uses this to figure out what plugin to run
            img_opts - a dict of additional options that specific to the target
                type we pass in via image_type
            fixed_params - a list of parameter keys that should not be
                           overridden by the --factory-parameter CLI
        @returns:
            A Builder() object from ImageFactory that contains information
            about the image building include state and progress.
        """
        # TODO: test the failure case where IF itself throws an exception
        #       ungracefully (missing a plugin for example)
        #       may need to still upload ozlog and remove the log handler
        if img_opts is None:
            img_opts = {}
        if fixed_params is None:
            fixed_params = []
        self.logger.debug('img_opts_pre_merge: %s' % img_opts)
        self._mergeFactoryParams(img_opts, fixed_params)
        self.logger.debug('img_opts_post_merge: %s' % img_opts)
        target = self.bd.builder_for_target_image(image_type,
                                                  image_id=base_id,
                                                  template=None,
                                                  parameters=img_opts)
        target.target_thread.join()
        self._checkImageState(target)
        return target

    def _buildConvert(self, format):
        """
        Build an image by converting the format using qemu-img. This is method
        enables a variety of formats like qcow, qcow2, vmdk, and vdi.

        @args:
            format - a string representing the image format, "qcow2"
        @returns
            a dict with some metadata about the image
        """
        self.logger.debug('converting an image to "%s"' % format)
        ofmt = format
        if format == 'vpc':
            ofmt = 'vhd'
        newimg = os.path.join(self.workdir, self.imgname + '.%s' % ofmt)
        cmd = ['/usr/bin/qemu-img', 'convert', '-f', 'raw', '-O',
               format, self.base_img.base_image.data, newimg]
        if format == 'qcow':
            cmd.insert(2, '-c')  # enable compression for qcow images
        if format == 'qcow2':
            # qemu-img changed its default behavior at some point to generate a
            # v3 image when the requested output format is qcow2.  We don't
            # want koji to output different formats based on the version of
            # qemu-img that happens to be on the builder. Here we use a function
            # inside of Image Factory that detects qemu-img behavior and adds
            # the correct options to ensure original "v2" compatibility
            cmd = qemu_convert_cmd(self.base_img.base_image.data, newimg, compress=True)
            # Factory does not use a full path - for consistency, force that here
            cmd[0] = '/usr/bin/qemu-img'
        conlog = os.path.join(self.workdir,
                              'qemu-img-%s-%s.log' % (format, self.arch))
        log_output(self.session, cmd[0], cmd, conlog,
                   self.getUploadDir(), logerror=1)
        return {'image': newimg}

    def handler(self, name, version, release, arch, target_info,
                build_tag, repo_info, inst_tree, opts=None):
        if not ozif_enabled:
            self.logger.error(
                "ImageFactory features require the following dependencies: "
                "pykickstart, imagefactory, oz and possibly python-hashlib")
            raise RefuseTask('ImageFactory functions not available')

        if opts is None:
            opts = {}
        self.arch = arch
        self.target_info = target_info
        self.repo_info = repo_info
        self.opts = opts
        self.formats = self._format_deps(opts.get('format'))

        # First, prepare the kickstart to use the repos we tell it
        kspath = self.fetchKickstart(build_tag=target_info['build_tag_name'])
        ks = self.prepareKickstart(kspath, inst_tree)
        kskoji = self.writeKickstart(ks,
                                     os.path.join(self.workdir, 'koji-%s-%i-base.ks' %
                                                  (self.target_info['build_tag_name'], self.id)))

        # auto-generate a TDL file and config dict for ImageFactory
        self.imgname = '%s-%s-%s.%s' % (name, version, release, self.arch)
        template = self.makeTemplate(self.imgname, inst_tree)
        self.logger.debug('oz template: %s' % template)
        config = self.makeConfig()
        self.logger.debug('IF config object: %s' % config)
        ApplicationConfiguration(configuration=config)

        tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % self.arch)
        with koji._open_text_file(tdl_path, 'wt') as tdl:
            tdl.write(template)
        self.uploadFile(tdl_path)

        # ImageFactory picks a port to the guest VM using a rolling integer.
        # This is a problem for concurrency, so we override the port it picks
        # here using the task ID. (not a perfect solution but good enough:
        # the likelihood of image tasks clashing here is very small)
        rm = ReservationManager()
        rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT)
        ozlogname = 'oz-%s.log' % self.arch
        self.ozlog = os.path.join(self.workdir, ozlogname)

        # invoke the image builds
        images = self.do_images(ks, template, inst_tree)
        images['raw']['tdl'] = os.path.basename(tdl_path),

        # structure the results to pass back to the hub:
        imgdata = {
            'arch': self.arch,
            'task_id': self.id,
            'logs': [ozlogname],
            'name': name,
            'version': version,
            'release': release,
            'rpmlist': [],
            'files': [os.path.basename(tdl_path),
                      os.path.basename(kspath),
                      os.path.basename(kskoji)]
        }
        # record the RPMs that were installed
        if not opts.get('scratch'):
            # fields = ('name', 'version', 'release', 'arch', 'epoch', 'size',
            #    'payloadhash', 'buildtime')
            icicle = xml.dom.minidom.parseString(images['raw']['icicle'])  # nosec
            self.logger.debug('ICICLE: %s' % images['raw']['icicle'])
            for p in icicle.getElementsByTagName('extra'):
                bits = p.firstChild.nodeValue.split(',')
                rpm = {
                    'name': bits[0],
                    'version': bits[1],
                    'release': bits[2],
                    'arch': bits[3],
                    # epoch is a special case, as usual
                    'size': int(bits[5]),
                    'payloadhash': bits[6],
                    'buildtime': int(bits[7])
                }
                if rpm['name'] in ['buildsys-build', 'gpg-pubkey']:
                    continue
                if bits[4] == '(none)':
                    rpm['epoch'] = None
                else:
                    rpm['epoch'] = int(bits[4])
                imgdata['rpmlist'].append(rpm)
            # TODO: hack to make this work for now, need to refactor
            br = BuildRoot(self.session, self.options, build_tag, self.arch,
                           self.id, repo_id=self.repo_info['id'])
            br.markExternalRPMs(imgdata['rpmlist'])

        # upload the results
        for format in (f for f in self.formats if self.formats[f]):
            newimg = images[format]['image']
            if ('ova' in format or format in ('raw-xz', 'liveimg-squashfs', 'tar-gz')):
                newname = self.imgname + '.' + format.replace('-', '.')
            elif 'vagrant' in format:
                # This embeds the vagrant target and the ".box" format in the name
                # Previously, based on filename, these looked like OVAs
                # This was confusing to many people
                newname = self.imgname + '.' + format + '.box'
            elif format == 'docker':
                newname = self.imgname + '.' + 'tar.xz'
            elif format == 'vpc':
                newname = self.imgname + '.' + 'vhd'
            else:
                newname = self.imgname + '.' + format
            if format != 'docker':
                lxml = images[format]['libvirt']
                imgdata['files'].append(os.path.basename(lxml))
                self.uploadFile(lxml)
            imgdata['files'].append(os.path.basename(newname))
            self.uploadFile(newimg, remoteName=newname)

        # no need to delete anything since self.workdir will get scrubbed
        return imgdata


class BuildIndirectionImageTask(OzImageTask):
    Methods = ['indirectionimage']

    # So, these are copied directly from the base image class
    # Realistically, we want to inherit methods from both BuildImageTask
    # and OzImageTask.
    # TODO: refactor - my initial suggestion would be to have OzImageTask
    #       be a child of BuildImageTask

    def initImageBuild(self, name, version, release, target_info, opts):
        """create a build object for this image build"""
        pkg_cfg = self.session.getPackageConfig(target_info['dest_tag_name'],
                                                name)
        self.logger.debug("%r" % pkg_cfg)
        if not opts.get('skip_tag') and not opts.get('scratch'):
            # Make sure package is on the list for this tag
            if pkg_cfg is None:
                raise koji.BuildError("package (image) %s not in list for tag %s" %
                                      (name, target_info['dest_tag_name']))
            elif pkg_cfg['blocked']:
                raise koji.BuildError("package (image)  %s is blocked for tag %s" %
                                      (name, target_info['dest_tag_name']))
        return self.session.host.initImageBuild(self.id,
                                                dict(name=name, version=version, release=release,
                                                     epoch=0))
    # END inefficient base image task method copies

    def fetchHubOrSCM(self, filepath, fileurl, build_tag):
        """
        Retrieve a file either from the hub or a remote scm

        If fileurl is None we assume we are being asked to retrieve from
        the hub and that filepath is relative to /mnt/koji/work.
        if fileurl contains a value we assume a remote SCM.

        If retrieving remote we assume that filepath is the file name and
        fileurl is the path in the remote SCM where that file can be found.

        @returns: absolute path to the retrieved file
        """
        # TODO: A small change to the base image build code could allow this method
        #       to be shared between both tasks.  I wanted this initial implementation
        #       to be entirely self contained.  Revisit if anyone feels like a refactor.
        self.logger.debug("filepath = %s" % filepath)
        if fileurl:
            scm = SCM(fileurl, allow_password=self.options.allow_password_in_scm_url)
            scm.assert_allowed(allowed=self.options.allowed_scms,
                               session=self.session,
                               by_config=self.options.allowed_scms_use_config,
                               by_policy=self.options.allowed_scms_use_policy,
                               policy_data={
                                   'user_id': self.taskinfo['owner'],
                                   'channel': self.session.getChannel(self.taskinfo['channel_id'],
                                                                      strict=True)['name'],
                                   'scratch': self.opts.get('scratch')
                               })
            self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
                               build_tag=build_tag, scratch=self.opts.get('scratch'))
            logfile = os.path.join(self.workdir, 'checkout.log')
            scmsrcdir = scm.checkout(self.workdir, self.session,
                                     self.getUploadDir(), logfile)
            self.run_callbacks("postSCMCheckout",
                               scminfo=scm.get_info(),
                               build_tag=build_tag,
                               scratch=self.opts.get('scratch'),
                               srcdir=scmsrcdir)
            final_path = os.path.join(scmsrcdir, os.path.basename(filepath))
        else:
            tops = dict([(k, getattr(self.options, k)) for k in ('topurl', 'topdir')])
            tops['tempdir'] = self.workdir
            final_path = os.path.join(self.workdir, os.path.basename(filepath))
            with koji.openRemoteFile(filepath, **tops) as remote_fileobj:
                with open(final_path, 'wb') as final_fileobj:
                    shutil.copyfileobj(remote_fileobj, final_fileobj)
        self.logger.debug('uploading retrieved file from here: %s' % final_path)
        self.uploadFile(final_path)  # upload the original ks file
        return final_path  # absolute path to the ks file

    def handler(self, opts):
        """Governing task for building an image with two other images using Factory Indirection"""
        # TODO: Add mode of operation where full build details are given for
        #       either base or utility or both, then spawn subtasks to do them first
        def _task_to_image(task_id):
            """ Take a task ID and turn it into an Image Factory Base Image object """
            pim = PersistentImageManager.default_manager()
            taskinfo = self.session.getTaskInfo(task_id)
            taskstate = koji.TASK_STATES[taskinfo['state']].lower()
            if taskstate != 'closed':
                raise koji.BuildError("Input task (%d) must be in closed state"
                                      " - current state is (%s)" %
                                      (task_id, taskstate))
            taskmethod = taskinfo['method']
            if taskmethod != "createImage":
                raise koji.BuildError("Input task method must be 'createImage'"
                                      " - actual method (%s)" %
                                      (taskmethod))
            result = self.session.getTaskResult(task_id)

            # This approach works for both scratch and saved/formal images
            # The downside is that we depend on the output file naming convention
            def _match_name(inlist, namere):
                for filename in inlist:
                    if re.search(namere, filename):
                        return filename
            task_diskimage = _match_name(result['files'], ".*qcow2$")
            task_tdl = _match_name(result['files'], "tdl.*xml")

            task_dir = os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(task_id))
            diskimage_full = os.path.join(task_dir, task_diskimage)
            tdl_full = os.path.join(task_dir, task_tdl)

            if not (os.path.isfile(diskimage_full) and os.path.isfile(tdl_full)):
                raise koji.BuildError(
                    "Missing TDL or qcow2 image for task (%d) - possible expired scratch build" %
                    (task_id))

            # The sequence to recreate a valid persistent image is as follows
            # Create a new BaseImage object
            factory_base_image = BaseImage()
            # Add it to the persistence layer
            pim.add_image(factory_base_image)
            # Now replace the data and template with the files referenced above
            # and mark it as a complete image
            # Factory doesn't attempt to modify a disk image after it is COMPLETE so
            # this will work safely on read-only NFS mounts
            factory_base_image.data = diskimage_full
            factory_base_image.template = koji._open_text_file(tdl_full).read()
            factory_base_image.status = 'COMPLETE'
            # Now save it
            pim.save_image(factory_base_image)

            # We can now reference this object directly or via its UUID in persistent storage
            return factory_base_image

        def _nvr_to_image(nvr, arch):
            """
            Take a build ID or NVR plus arch and turn it into
            an Image Factory Base Image object
            """
            pim = PersistentImageManager.default_manager()
            build = self.session.getBuild(nvr)
            if not build:
                raise koji.BuildError("Could not find build for (%s)" % (nvr))

            buildarchives = self.session.listArchives(build['id'])
            if not buildarchives:
                raise koji.BuildError("Could not retrieve archives for build (%s) from NVR (%s)" %
                                      (build['id'], nvr))

            buildfiles = [x['filename'] for x in buildarchives]
            builddir = koji.pathinfo.imagebuild(build)

            def _match_name(inlist, namere):
                for filename in inlist:
                    if re.search(namere, filename):
                        return filename

            build_diskimage = _match_name(buildfiles, r".*%s\.qcow2$" % (arch))
            build_tdl = _match_name(buildfiles, r"tdl.%s\.xml" % (arch))

            diskimage_full = os.path.join(builddir, build_diskimage)
            tdl_full = os.path.join(builddir, build_tdl)

            if not (os.path.isfile(diskimage_full) and os.path.isfile(tdl_full)):
                raise koji.BuildError("Missing TDL (%s) or qcow2 (%s) image for image (%s)"
                                      " - this should never happen" %
                                      (build_tdl, build_diskimage, nvr))

            # The sequence to recreate a valid persistent image is as follows
            # Create a new BaseImage object
            factory_base_image = BaseImage()
            # Add it to the persistence layer
            pim.add_image(factory_base_image)
            # Now replace the data and template with the files referenced above
            # and mark it as a complete image
            # Factory doesn't attempt to modify a disk image after it is COMPLETE so
            # this will work safely on read-only NFS mounts
            factory_base_image.data = diskimage_full
            factory_base_image.template = koji._open_text_file(tdl_full).read()
            factory_base_image.status = 'COMPLETE'
            # Now save it
            pim.save_image(factory_base_image)

            # We can now reference this object directly or via its UUID in persistent storage
            return factory_base_image

        if opts is None:
            opts = {}
        self.opts = opts

        config = self.makeConfig()
        self.logger.debug('IF config object: %s' % config)
        ApplicationConfiguration(configuration=config)

        ozlogname = 'oz-indirection.log'
        ozlog = os.path.join(self.workdir, ozlogname)
        # END shared code

        fhandler = logging.FileHandler(ozlog)
        bd = BuildDispatcher()
        tlog = logging.getLogger()
        tlog.setLevel(logging.DEBUG)
        tlog.addHandler(fhandler)

        # TODO: Copy-paste from BaseImage - refactor
        target_info = self.session.getBuildTarget(opts['target'], strict=True)

        name = opts['name']
        version = opts['version']
        release = opts['release']

        # TODO: Another mostly copy-paste
        if '-' in version:
            raise koji.ApplianceError('The Version may not have a hyphen')
        if release and '-' in release:
            raise koji.ApplianceError('The Release may not have a hyphen')

        indirection_template = self.fetchHubOrSCM(opts.get('indirection_template'),
                                                  opts.get('indirection_template_url'),
                                                  target_info['build_tag_name'])

        self.logger.debug('Got indirection template %s' % (indirection_template))

        try:
            if opts['utility_image_build']:
                utility_factory_image = _nvr_to_image(opts['utility_image_build'], opts['arch'])
            else:
                utility_factory_image = _task_to_image(int(opts['utility_image_task']))

            if opts['base_image_build']:
                base_factory_image = _nvr_to_image(opts['base_image_build'], opts['arch'])
            else:
                base_factory_image = _task_to_image(int(opts['base_image_task']))
        except Exception as e:
            self.logger.exception(e)
            raise

        # OK - We have a template and two input images - lets build
        bld_info = None
        if not opts['scratch']:
            bld_info = self.initImageBuild(name, version, release,
                                           target_info, opts)

        try:
            return self._do_indirection(opts, base_factory_image, utility_factory_image,
                                        indirection_template, tlog, ozlog, fhandler,
                                        bld_info, target_info, bd)
        except Exception:
            if not opts.get('scratch'):
                # scratch builds do not get imported
                if bld_info:
                    self.session.host.failBuild(self.id, bld_info['id'])
            # reraise the exception
            raise

    def _do_indirection(self, opts, base_factory_image, utility_factory_image,
                        indirection_template, tlog, ozlog, fhandler, bld_info,
                        target_info, bd):
        # TODO: The next several lines are shared with the handler for other Factory tasks
        #       refactor in such a way that this can be a helper in OzImageTask

        # ImageFactory picks a port to the guest VM using a rolling integer.
        # This is a problem for concurrency, so we override the port it picks
        # here using the task ID. (not a perfect solution but good enough:
        # the likelihood of image tasks clashing here is very small)
        rm = ReservationManager()
        rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT)

        utility_customizations = koji._open_text_file(indirection_template).read()
        results_loc = opts.get('results_loc', None)
        if results_loc[0] != "/":
            results_loc = "/" + results_loc
        params = {'utility_image': str(utility_factory_image.identifier),
                  'utility_customizations': utility_customizations,
                  'results_location': results_loc}
        random.seed()  # necessary to ensure a unique mac address
        try:
            try:
                # Embedded deep debug option - if template is just the string MOCK
                # skip the actual build and create a mock target image instead
                if utility_customizations.strip() == "MOCK":
                    target = Builder()
                    target_image = TargetImage()
                    pim = PersistentImageManager.default_manager()
                    pim.add_image(target_image)
                    target.target_image = target_image
                    with koji._open_text_file(target_image.data, "wt") as f:
                        f.write("Mock build from task ID: %s" % self.id)
                    target_image.status = 'COMPLETE'
                else:
                    target = bd.builder_for_target_image('indirection',
                                                         image_id=base_factory_image.identifier,
                                                         parameters=params)
                    target.target_thread.join()
            except Exception as e:
                self.logger.debug("Exception encountered during target build")
                self.logger.exception(e)
        finally:
            # upload log even if we failed to help diagnose an issue
            tlog.removeHandler(fhandler)
            self.uploadFile(ozlog)
        self.logger.debug('Target image results: %s' % target.target_image.status)

        if target.target_image.status == 'FAILED':
            # TODO abort when a task is CANCELLED
            if not self.session.checkUpload('', os.path.basename(ozlog)):
                tlog.removeHandler(fhandler)
                self.uploadFile(ozlog)
            raise koji.ApplianceError('Image status is %s: %s' %
                                      (target.target_image.status,
                                       target.target_image.status_detail))

        self.uploadFile(target.target_image.data, remoteName=os.path.basename(results_loc))

        myresults = {}
        myresults['task_id'] = self.id
        myresults['files'] = [os.path.basename(results_loc)]
        myresults['logs'] = [os.path.basename(ozlog)]
        myresults['arch'] = opts['arch']
        # TODO: This should instead track the two input images: base and utility
        myresults['rpmlist'] = []

        # This is compatible with some helper methods originally implemented for the base
        # image build.  In the original usage, the dict contains an entry per build arch
        # TODO: If adding multiarch support, keep this in mind
        results = {str(self.id): myresults}
        self.logger.debug('Image Results for hub: %s' % results)

        if opts['scratch']:
            self.session.host.moveImageBuildToScratch(self.id, results)
        else:
            self.session.host.completeImageBuild(self.id, bld_info['id'],
                                                 results)

        # tag it
        if not opts.get('scratch') and not opts.get('skip_tag'):
            tag_task_id = self.session.host.subtask(method='tagBuild',
                                                    arglist=[target_info['dest_tag'],
                                                             bld_info['id'], False, None, True],
                                                    label='tag', parent=self.id, arch='noarch')
            self.wait(tag_task_id)

        # report results
        report = ''
        if opts.get('scratch'):
            respath = ', '.join(
                [os.path.join(koji.pathinfo.work(),
                              koji.pathinfo.taskrelpath(tid)) for tid in [self.id]])
            report += 'Scratch '
        else:
            respath = koji.pathinfo.imagebuild(bld_info)
        report += 'image build results in: %s' % respath
        return report


class RebuildSRPM(BaseBuildTask):

    Methods = ['rebuildSRPM']
    _taskWeight = 1.0

    def checkHost(self, hostdata):
        tag = self.params[1]
        return self.checkHostArch(tag, hostdata)

    def handler(self, srpm, build_tag, opts=None):
        if opts is None:
            opts = {}
        repo_id = opts.get('repo_id')
        if not repo_id:
            raise koji.BuildError("A repo id must be provided")

        repo_info = self.session.repoInfo(repo_id, strict=True)
        event_id = repo_info['create_event']
        build_tag = self.session.getTag(build_tag, strict=True, event=event_id)

        rootopts = {'install_group': 'srpm-build', 'repo_id': repo_id}
        br_arch = self.find_arch('noarch', self.session.host.getHost(
        ), self.session.getBuildConfig(build_tag['id'], event=event_id))
        broot = BuildRoot(self.session, self.options,
                          build_tag['id'], br_arch, self.id, **rootopts)
        broot.workdir = self.workdir

        self.logger.debug("Initializing buildroot")
        broot.init()

        # Setup files and directories for SRPM rebuild
        # We can't put this under the mock homedir because that directory
        # is completely blown away and recreated on every mock invocation
        srpmdir = broot.tmpdir() + '/srpm'
        koji.ensuredir(srpmdir)
        uploadpath = self.getUploadDir()

        fn = self.localPath("work/%s" % srpm)
        if not os.path.exists(fn):
            raise koji.BuildError("Input SRPM file missing: %s" % fn)
        shutil.copy(fn, srpmdir)

        # rebuild srpm
        self.logger.debug("Running srpm rebuild")
        br_srpm_path = os.path.join(broot.path_without_to_within(srpmdir), os.path.basename(srpm))
        broot.rebuild_srpm(br_srpm_path)

        srpms = glob.glob('%s/*.src.rpm' % broot.resultdir())
        if len(srpms) == 0:
            raise koji.BuildError("No srpms found in %s" % srpmdir)
        elif len(srpms) > 1:
            raise koji.BuildError("Multiple srpms found in %s: %s" % (srpmdir, ", ".join(srpms)))
        else:
            srpm = srpms[0]

        # check srpm name
        h = koji.get_rpm_header(srpm)
        name = koji.get_header_field(h, 'name')
        version = koji.get_header_field(h, 'version')
        release = koji.get_header_field(h, 'release')
        srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals()
        if srpm_name != os.path.basename(srpm):
            raise koji.BuildError('srpm name mismatch: %s != %s' %
                                  (srpm_name, os.path.basename(srpm)))

        # upload srpm and return
        self.uploadFile(srpm)

        brootid = broot.id
        log_files = list(broot.logs)

        broot.expire()

        return {
            'srpm': "%s/%s" % (uploadpath, srpm_name),
            'logs': ["%s/%s" % (uploadpath, f) for f in log_files],
            'brootid': brootid,
            'source': {
                'source': os.path.basename(srpm),
                'url': os.path.basename(srpm),
            }
        }


class BuildSRPMFromSCMTask(BaseBuildTask):

    Methods = ['buildSRPMFromSCM']
    _taskWeight = 1.0

    def spec_sanity_checks(self, filename):
        spec = koji._open_text_file(filename).read()
        for tag in ("Packager", "Distribution", "Vendor"):
            if re.match("%s:" % tag, spec, re.M):
                raise koji.BuildError("%s is not allowed to be set in spec file" % tag)
        for tag in ("packager", "distribution", "vendor"):
            if re.match(r"%%define\s+%s\s+" % tag, spec, re.M):
                raise koji.BuildError("%s is not allowed to be defined in spec file" % tag)

    def patch_scm_source(self, sourcedir, logfile, opts):
        # override if desired
        pass

    def checkHost(self, hostdata):
        tag = self.params[1]
        return self.checkHostArch(tag, hostdata)

    def handler(self, url, build_tag, opts=None):
        if opts is None:
            opts = {}
        # will throw a BuildError if the url is invalid
        scm = SCM(url, allow_password=self.options.allow_password_in_scm_url)
        scm.assert_allowed(allowed=self.options.allowed_scms,
                           session=self.session,
                           by_config=self.options.allowed_scms_use_config,
                           by_policy=self.options.allowed_scms_use_policy,
                           policy_data={
                               'user_id': self.taskinfo['owner'],
                               'channel': self.session.getChannel(self.taskinfo['channel_id'],
                                                                  strict=True)['name'],
                               'scratch': opts.get('scratch')
                           })
        repo_id = opts.get('repo_id')
        if not repo_id:
            raise koji.BuildError("A repo id must be provided")

        repo_info = self.session.repoInfo(repo_id, strict=True)
        event_id = repo_info['create_event']
        build_tag = self.session.getTag(build_tag, strict=True, event=event_id)

        # need DNS in the chroot because "make srpm" may need to contact
        # a SCM or lookaside cache to retrieve the srpm contents
        rootopts = {'install_group': 'srpm-build',
                    'setup_dns': True,
                    'repo_id': repo_id}
        if self.options.scm_credentials_dir is not None and os.path.isdir(
                self.options.scm_credentials_dir):
            rootopts['bind_opts'] = {'dirs': {self.options.scm_credentials_dir: '/credentials', }}
            # Force internal_dev_setup back to true because bind_opts is used to turn it off
            rootopts['internal_dev_setup'] = True
        br_arch = self.find_arch('noarch', self.session.host.getHost(
        ), self.session.getBuildConfig(build_tag['id'], event=event_id))
        broot = BuildRoot(self.session, self.options,
                          build_tag['id'], br_arch, self.id, **rootopts)
        broot.workdir = self.workdir

        self.logger.debug("Initializing buildroot")
        broot.init()

        # Setup files and directories for SRPM creation
        # We can't put this under the mock homedir because that directory
        # is completely blown away and recreated on every mock invocation
        scmdir = broot.tmpdir() + '/scmroot'
        koji.ensuredir(scmdir)
        logfile = self.workdir + '/checkout.log'
        uploadpath = self.getUploadDir()

        self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(),
                           build_tag=build_tag, scratch=opts.get('scratch'),
                           buildroot=broot)
        # Check out spec file, etc. from SCM
        sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile)
        self.run_callbacks("postSCMCheckout",
                           scminfo=scm.get_info(),
                           build_tag=build_tag,
                           scratch=opts.get('scratch'),
                           srcdir=sourcedir,
                           buildroot=broot)
        # get the source before chown, git > 2.35.2 would refuse to that later
        source = scm.get_source()
        # chown the sourcedir and everything under it to the mockuser
        # so we can build the srpm as non-root
        uid = pwd.getpwnam(self.options.mockuser)[2]
        # rpmbuild seems to complain if it's running in the "mock" group but
        # files are in a different group
        gid = grp.getgrnam('mock')[2]
        self.chownTree(scmdir, uid, gid)

        # Hook for patching spec file in place
        self.patch_scm_source(sourcedir, logfile, opts)

        # Find and verify that there is only one spec file.
        spec_files = glob.glob("%s/*.spec" % sourcedir)
        if not spec_files and self.options.support_rpm_source_layout:
            # also check SPECS dir
            spec_files = glob.glob("%s/SPECS/*.spec" % sourcedir)
        if len(spec_files) == 0:
            raise koji.BuildError("No spec file found")
        elif len(spec_files) > 1:
            # If there are multiple spec files, check whether one of them
            # matches the SCM repo name
            scm_spec_options = (
                "%s/%s.spec" % (sourcedir, os.path.basename(sourcedir)),
                "%s/SPECS/%s.spec" % (sourcedir, os.path.basename(sourcedir)),
            )

            spec_file = None
            for scm_spec in scm_spec_options:
                if scm_spec in spec_files:
                    # We have a match, so use this one.
                    spec_file = scm_spec
                    break

            if not spec_file:
                # We didn't find an exact match, so throw an error
                raise koji.BuildError("Multiple spec files found but none is matching "
                                      "SCM checkout dir name: %s" % spec_files)
        else:
            spec_file = spec_files[0]

        # Run spec file sanity checks.  Any failures will throw a BuildError
        self.spec_sanity_checks(spec_file)

        # build srpm
        self.logger.debug("Running srpm build")
        broot.build_srpm(spec_file, sourcedir, scm.source_cmd)

        srpms = glob.glob('%s/*.src.rpm' % broot.resultdir())
        if len(srpms) == 0:
            raise koji.BuildError("No srpms found in %s" % sourcedir)
        elif len(srpms) > 1:
            raise koji.BuildError("Multiple srpms found in %s: %s" % (sourcedir, ", ".join(srpms)))
        else:
            srpm = srpms[0]

        # check srpm name
        h = koji.get_rpm_header(srpm)
        name = koji.get_header_field(h, 'name')
        version = koji.get_header_field(h, 'version')
        release = koji.get_header_field(h, 'release')
        srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals()
        if srpm_name != os.path.basename(srpm):
            raise koji.BuildError('srpm name mismatch: %s != %s' %
                                  (srpm_name, os.path.basename(srpm)))

        # upload srpm and return
        self.uploadFile(srpm)

        brootid = broot.id
        log_files = list(broot.logs)

        broot.expire()

        return {'srpm': "%s/%s" % (uploadpath, srpm_name),
                'logs': ["%s/%s" % (uploadpath, f) for f in log_files],
                'brootid': brootid,
                'source': source,
                }


class TagNotificationTask(BaseTaskHandler):
    Methods = ['tagNotification']

    _taskWeight = 0.1

    message_templ = \
        """From: %(from_addr)s\r
Subject: %(nvr)s %(result)s %(operation)s by %(user_name)s\r
To: %(to_addrs)s\r
X-Koji-Package: %(pkg_name)s\r
X-Koji-NVR: %(nvr)s\r
X-Koji-Draft: %(draft)s\r
X-Koji-User: %(user_name)s\r
X-Koji-Status: %(status)s\r
%(tag_headers)s\r
\r
Package: %(pkg_name)s\r
NVR: %(nvr)s\r
User: %(user_name)s\r
Status: %(status)s\r
%(operation_details)s\r
%(nvr)s %(result)s %(operation)s by %(user_name)s\r
%(failure_info)s\r
"""

    def handler(self, recipients, is_successful, tag_info, from_info,
                build_info, user_info, ignore_success=None, failure_msg=''):
        if len(recipients) == 0:
            self.logger.debug('task %i: no recipients, not sending notifications', self.id)
            return

        if ignore_success and is_successful:
            self.logger.debug(
                'task %i: tag operation successful and ignore success is true, '
                'not sending notifications', self.id)
            return

        build = self.session.getBuild(build_info)
        user = self.session.getUser(user_info)
        pkg_name = build['package_name']
        nvr = koji.buildLabel(build)
        draft = build.get('draft', False)
        user_name = user['name']

        from_addr = self.options.from_addr
        to_addrs = ', '.join(recipients)

        operation = '%(action)s'
        operation_details = 'Tag Operation: %(action)s\r\n'
        tag_headers = ''
        if from_info:
            from_tag = self.session.getTag(from_info)
            from_tag_name = from_tag['name']
            operation += ' from %s' % from_tag_name
            operation_details += 'From Tag: %s\r\n' % from_tag_name
            tag_headers += 'X-Koji-Tag: %s' % from_tag_name
            action = 'untagged'
        if tag_info:
            tag = self.session.getTag(tag_info)
            tag_name = tag['name']
            operation += ' into %s' % tag_name
            operation_details += 'Into Tag: %s\r\n' % tag_name
            if tag_headers:
                tag_headers += '\r\n'
            tag_headers += 'X-Koji-Tag: %s' % tag_name
            action = 'tagged'
        if tag_info and from_info:
            action = 'moved'
        operation = operation % locals()
        operation_details = operation_details % locals()

        if is_successful:
            result = 'successfully'
            status = 'complete'
            failure_info = ''
        else:
            result = 'unsuccessfully'
            status = 'failed'
            failure_info = "Operation failed with the error:\r\n    %s\r\n" % failure_msg

        message = self.message_templ % locals()
        # ensure message is in UTF-8
        message = koji.fixEncoding(message)
        # binary for python3
        if six.PY3:
            message = message.encode('utf8')
        server = smtplib.SMTP(self.options.smtphost)
        if self.options.smtp_user is not None and self.options.smtp_pass is not None:
            server.login(self.options.smtp_user, self.options.smtp_pass)
        # server.set_debuglevel(True)

        server.sendmail(from_addr, recipients, message)
        server.quit()

        return 'sent notification of tag operation %i to: %s' % (self.id, to_addrs)


class BuildNotificationTask(BaseTaskHandler):
    Methods = ['buildNotification']

    _taskWeight = 0.1

    # XXX externalize these templates somewhere
    subject_templ = "Package: %(build_nvr)s Tag: %(dest_tag)s Status: %(status)s " \
                    "Built by: %(build_owner)s"
    message_templ = \
        """From: %(from_addr)s\r
Subject: %(subject)s\r
To: %(to_addrs)s\r
X-Koji-Tag: %(dest_tag)s\r
X-Koji-Package: %(build_pkg_name)s\r
X-Koji-Builder: %(build_owner)s\r
X-Koji-Status: %(status)s\r
X-Koji-Draft: %(draft)s\r
\r
Package: %(build_nvr)s\r
Tag: %(dest_tag)s\r
Status: %(status)s%(cancel_info)s\r
Built by: %(build_owner)s\r
ID: %(build_id)i\r
Started: %(creation_time)s\r
Finished: %(completion_time)s\r
%(changelog)s\r
%(failure)s\r
%(output)s\r
Task Info: %(weburl)s/taskinfo?taskID=%(task_id)i\r
Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r
"""

    def _getTaskData(self, task_id, data=None):
        if not data:
            data = {}
        taskinfo = self.session.getTaskInfo(task_id)

        if not taskinfo:
            # invalid task_id
            return data

        if taskinfo['host_id']:
            hostinfo = self.session.getHost(taskinfo['host_id'])
        else:
            hostinfo = None

        result = None
        try:
            result = self.session.getTaskResult(task_id)
        except Exception:
            excClass, result = sys.exc_info()[:2]
            if hasattr(result, 'faultString'):
                result = result.faultString
            else:
                result = '%s: %s' % (excClass.__name__, result)
            result = result.strip()
            # clear the exception, since we're just using
            # it for display purposes
            try:
                sys.exc_clear()
            except AttributeError:
                # sys.exc_clear() is obsolete in Python 3
                pass
        if not result:
            result = 'Unknown'

        logs, rpms, srpms, misc = [], [], [], []
        files_data = self.session.listTaskOutput(task_id, all_volumes=True)
        for filename in files_data:
            if filename.endswith('.log'):
                logs += [(filename, volume) for volume in files_data[filename]]
            # all rpms + srpms are expected to be in builddir
            elif filename.endswith('.src.rpm'):
                srpms.append(filename)
            elif filename.endswith('.rpm'):
                rpms.append(filename)
            else:
                misc += [(filename, volume) for volume in files_data[filename]]

        # sort by volumes and filenames
        logs.sort(key=lambda x: (x[1], x[0]))
        misc.sort(key=lambda x: (x[1], x[0]))
        rpms.sort()

        data[task_id] = {}
        data[task_id]['id'] = taskinfo['id']
        data[task_id]['method'] = taskinfo['method']
        data[task_id]['arch'] = taskinfo['arch']
        data[task_id]['build_arch'] = taskinfo['label'] or ''
        data[task_id]['host'] = hostinfo and hostinfo['name'] or None
        data[task_id]['state'] = koji.TASK_STATES[taskinfo['state']].lower()
        data[task_id]['result'] = result
        data[task_id]['request'] = self.session.getTaskRequest(task_id)
        data[task_id]['logs'] = logs
        data[task_id]['rpms'] = rpms
        data[task_id]['srpms'] = srpms
        data[task_id]['misc'] = misc

        children = self.session.getTaskChildren(task_id)
        for child in children:
            data = self._getTaskData(child['id'], data)
        return data

    def handler(self, recipients, build, target, weburl):
        if len(recipients) == 0:
            self.logger.debug('task %i: no recipients, not sending notifications', self.id)
            return

        build_pkg_name = build['package_name']
        build_pkg_evr = '%s%s-%s' % \
                        ((build['epoch'] and str(build['epoch']) + ':' or ''),
                         build['version'],
                         build['release'])
        build_nvr = koji.buildLabel(build)
        build_id = build['id']
        build_owner = build['owner_name']
        draft = build.get('draft', False)
        # target comes from session.py:_get_build_target()
        dest_tag = None
        if target is not None:
            dest_tag = target['dest_tag_name']
        status = koji.BUILD_STATES[build['state']].lower()
        creation_time = koji.formatTimeLong(build['creation_ts'])
        completion_time = koji.formatTimeLong(build['completion_ts'])
        task_id = build['task_id']

        task_data = self._getTaskData(task_id)

        cancel_info = ''
        failure_info = ''
        if build['state'] == koji.BUILD_STATES['CANCELED']:
            # The owner of the buildNotification task is the one
            # who canceled the task, it turns out.
            this_task = self.session.getTaskInfo(self.id)
            if this_task['owner']:
                canceler = self.session.getUser(this_task['owner'])
                cancel_info = "\r\nCanceled by: %s" % canceler['name']
        elif build['state'] == koji.BUILD_STATES['FAILED']:
            failure_data = task_data[task_id]['result']
            failed_hosts = ['%s (%s)' % (task['host'], task['arch'])
                            for task in task_data.values()
                            if task['host'] and task['state'] == 'failed']
            failure_info = "\r\n%s (%d) failed on %s:\r\n  %s" % (build_nvr, build_id,
                                                                  ', '.join(failed_hosts),
                                                                  failure_data)

        failure = failure_info or cancel_info or ''

        tasks = {'failed': [task for task in task_data.values() if task['state'] == 'failed'],
                 'canceled': [task for task in task_data.values() if task['state'] == 'canceled'],
                 'closed': [task for task in task_data.values() if task['state'] == 'closed']}

        srpms = []
        for taskinfo in task_data.values():
            for srpmfile in taskinfo['srpms']:
                srpms.append(srpmfile)
        srpms = sorted(self.uniq(srpms))

        if srpms:
            output = "SRPMS:\r\n"
            for srpm in srpms:
                output += "  %s" % srpm
            output += "\r\n\r\n"
        else:
            output = ''

        pathinfo = koji.PathInfo(topdir=self.options.topurl)
        buildurl = pathinfo.build(build)
        # list states here to make them go in the correct order
        for task_state in ['failed', 'canceled', 'closed']:
            if tasks[task_state]:
                output += "%s tasks:\r\n" % task_state.capitalize()
                output += "%s-------\r\n\r\n" % ("-" * len(task_state))
                for task in tasks[task_state]:
                    output += "Task %s" % task['id']
                    if task['host']:
                        output += " on %s\r\n" % task['host']
                    else:
                        output += "\r\n"
                    output += "Task Type: %s\r\n" % koji.taskLabel(task)
                    if task['logs']:
                        output += "logs:\r\n"
                        for (file_, volume) in task['logs']:
                            if tasks[task_state] != 'closed':
                                output += " %s/getfile?taskID=%s&name=%s&volume=%s\r\n" % (
                                    weburl, task['id'], file_, volume)
                            else:
                                output += " %s\r\n" % '/'.join([buildurl, 'data', 'logs',
                                                                task['build_arch'], file_])
                    if task['rpms']:
                        output += "rpms:\r\n"
                        for file_ in task['rpms']:
                            output += " %s\r\n" % '/'.join([buildurl, task['build_arch'], file_])
                    if task['misc']:
                        output += "misc:\r\n"
                        for (file_, volume) in task['misc']:
                            output += " %s/getfile?taskID=%s&name=%s&volume=%s\r\n" % (
                                weburl, task['id'], file_, volume)
                    output += "\r\n"
                output += "\r\n"

        changelog = koji.util.formatChangelog(self.session.getChangelogEntries(
            build_id, queryOpts={'limit': 3})).replace("\n", "\r\n")
        if changelog:
            changelog = "Changelog:\r\n%s" % changelog

        from_addr = self.options.from_addr
        to_addrs = ', '.join(recipients)
        subject = self.subject_templ % locals()
        message = self.message_templ % locals()
        # ensure message is in UTF-8
        message = koji.fixEncoding(message)
        # binary for python3
        if six.PY3:
            message = message.encode('utf8')
        server = smtplib.SMTP(self.options.smtphost)
        if self.options.smtp_user is not None and self.options.smtp_pass is not None:
            server.login(self.options.smtp_user, self.options.smtp_pass)
        # server.set_debuglevel(True)
        server.sendmail(from_addr, recipients, message)
        server.quit()

        return 'sent notification of build %i to: %s' % (build_id, to_addrs)

    def uniq(self, items):
        """Remove duplicates from the list of items, and sort the list."""
        m = dict(zip(items, [1] * len(items)))
        s = sorted(to_list(m.keys()))
        return s


class NewRepoTask(BaseTaskHandler):
    Methods = ['newRepo']
    _taskWeight = 0.1

    def copy_arch_repo(self, src_repo_id, src_repo_path, repo_id, arch):
        """Copy repodata, return False if it fails"""
        dst_repodata = joinpath(self.workdir, arch, 'repodata')
        src_repodata = joinpath(src_repo_path, arch, 'repodata')
        try:
            # copy repodata
            self.logger.debug('Copying repodata %s to %s' % (src_repodata, dst_repodata))
            if os.path.exists(src_repodata):
                # symlink=True is not needed as they are no part of arch repodir
                shutil.copytree(src_repodata, dst_repodata)
            uploadpath = self.getUploadDir()
            files = []
            for f in os.listdir(dst_repodata):
                files.append(f)
                self.session.uploadWrapper('%s/%s' % (dst_repodata, f), uploadpath, f)
            return [uploadpath, files]
        except Exception as ex:
            self.logger.warning("Copying repo %i to %i failed. %r" % (src_repo_id, repo_id, ex))
            # Try to remove potential leftovers and fail if there is some problem
            koji.util.rmtree(dst_repodata, self.logger)
            return False

    def check_repo(self, src_repo_path, dst_repo_path, src_repo, dst_repo, opts):
        """Check if oldrepo is reusable as is and can be directly copied"""
        # with_src, debuginfo, pkglist, blocklist, grouplist
        # We're ignoring maven support here. It is handled in repo_init which is called
        # always, so it doesn't affect efficiency of pre-cloning rpm repos.
        if not src_repo_path:
            self.logger.debug("Source repo wasn't found")
            return False
        if not os.path.isdir(src_repo_path):
            self.logger.debug("Source repo doesn't exist %s" % src_repo_path)
            return False
        try:
            repo_json = koji.load_json(joinpath(src_repo_path, 'repo.json'))
            for key in ('with_debuginfo', 'with_src', 'with_separate_src'):
                if repo_json.get(key, False) != opts.get(key, False):
                    return False
        except IOError:
            self.logger.debug("Can't open repo.json for repo {repo_info['id']}")
            return False

        # compare comps if they exist
        src_comps_path = joinpath(src_repo_path, 'groups/comps.xml')
        dst_comps_path = joinpath(dst_repo_path, 'groups/comps.xml')
        src_exists = os.path.exists(src_comps_path)
        if src_exists != os.path.exists(dst_comps_path):
            self.logger.debug("Comps exists only in one repo")
            return False
        if src_exists and not filecmp.cmp(src_comps_path, dst_comps_path, shallow=False):
            self.logger.debug("Comps differs")
            return False

        # if there is any external repo, don't trust the repodata
        if self.session.getExternalRepoList(src_repo['tag_id'], event=src_repo['create_event']):
            self.logger.debug("Source repo use external repos")
            return False
        if self.session.getExternalRepoList(dst_repo['tag_id'], event=dst_repo['create_event']):
            self.logger.debug("Destination repo use external repos")
            return False

        self.logger.debug('Repo test passed')
        return True

    def check_arch_repo(self, src_repo_path, dst_repo_path, arch):
        """More checks based on architecture content"""
        for fname in ('blocklist', 'pkglist'):
            src_file = joinpath(src_repo_path, arch, fname)
            dst_file = joinpath(dst_repo_path, arch, fname)
            # both must non/exist
            if not os.path.exists(src_file) or not os.path.exists(dst_file):
                self.logger.debug("%s doesn't exit in one of the repos" % fname)
                return False
            # content must be same
            if not filecmp.cmp(src_file, dst_file, shallow=False):
                self.logger.debug('%s differs' % fname)
                return False
        self.logger.debug('Arch repo test passed %s' % arch)
        return True

    def get_old_repo(self, tinfo):
        # oldrepo is not used if createrepo_update is not set, so don't waste time
        if not self.options.createrepo_update:
            return None, None

        # see if we can find a previous repo to update from
        oldrepo = self.session.getRepo(tinfo['id'])
        oldrepo_path = None
        if oldrepo:
            oldrepo_path = koji.pathinfo.repo(oldrepo['id'], tinfo['name'])
            oldrepo['tag_id'] = tinfo['id']
            return oldrepo, oldrepo_path

        # If there is no old repo, try to find a suitable repo from a related tag
        # Check configured hints first - can be a single tag or list
        hint = tinfo['extra'].get('repo.oldrepo_hint', [])
        if not isinstance(hint, list):
            hint = [hint]
        for _tag in hint:
            reftag = self.session.getTag(_tag, strict=False)
            if not reftag:
                self.logger.warning('oldrepo_hint refers to invalid tag: %s', _tag)
                continue
            oldrepo = self.session.getRepo(reftag['id'])
            if oldrepo:
                oldrepo_path = koji.pathinfo.repo(oldrepo['id'], reftag['name'])
                oldrepo['tag_id'] = reftag['id']
                return oldrepo, oldrepo_path

        # Otherwise consider tags in the inheritance
        parents = self.session.getFullInheritance(tinfo['id'])
        # we care about best candidate which should be (not necessarily)
        # something on higher levels. Sort tags according to depth.
        for link in sorted(parents, key=lambda x: x['currdepth']):
            oldrepo = self.session.getRepo(link['parent_id'])
            if oldrepo:
                parenttag = self.session.getTag(link['parent_id'])
                oldrepo_path = koji.pathinfo.repo(oldrepo['id'], parenttag['name'])
                oldrepo['tag_id'] = parenttag['id']
                return oldrepo, oldrepo_path

        # otherwise no match
        return None, None

    def handler(self, tag, event=None, src=None, debuginfo=None, separate_src=None, opts=None):
        tinfo = self.session.getTag(tag, strict=True, event=event)

        # handle deprecated opts
        _opts = {}
        if src is not None:
            _opts['src'] = bool(src)
        if debuginfo is not None:
            _opts['debuginfo'] = bool(debuginfo)
        if separate_src is not None:
            _opts['separate_src'] = bool(separate_src)
        if _opts:
            if opts is not None:
                raise koji.ParameterError('opts parameter cannot be combined with legacy options')
            self.logger.warning('The src, debuginfo, and separate_src parameters for newRepo '
                                'tasks are deprecated. Use the opts parameter.')
            opts = _opts

        # check for fs access before we try calling repoInit
        top_repos_dir = joinpath(self.options.topdir, "repos")
        if not os.path.isdir(top_repos_dir):
            # missing or incorrect mount?
            # refuse and let another host try
            raise RefuseTask("No access to repos dir %s" % top_repos_dir)

        # call repoInit
        kwargs = {'opts': opts, 'task_id': self.id}
        if event is not None:
            kwargs['event'] = event
        repo_id, event_id = self.session.host.repoInit(tinfo['id'], **kwargs)

        path = koji.pathinfo.repo(repo_id, tinfo['name'])
        if not os.path.isdir(path):
            raise koji.GenericError("Repo directory missing: %s" % path)
        arches = []
        for fn in os.listdir(path):
            if fn != 'groups' and os.path.isfile("%s/%s/pkglist" % (path, fn)):
                arches.append(fn)

        oldrepo, oldrepo_path = self.get_old_repo(tinfo)
        newrepo_path = koji.pathinfo.repo(repo_id, tinfo['name'])
        newrepo = {'tag_id': tinfo['id'], 'create_event': event_id}
        if self.options.copy_old_repodata:
            possibly_clonable = self.check_repo(oldrepo_path, newrepo_path,
                                                oldrepo, newrepo, kwargs)
        else:
            possibly_clonable = False
        subtasks = {}
        data = {}
        cloned_archs = []
        for arch in arches:
            if possibly_clonable and self.check_arch_repo(oldrepo_path, newrepo_path, arch):
                result = self.copy_arch_repo(oldrepo['id'], oldrepo_path, repo_id, arch)
                if result:
                    data[arch] = result
                    cloned_archs.append(arch)
                    continue
            # if we can't copy old repo directly, trigger normal createrepo
            arglist = [repo_id, arch, oldrepo]
            subtasks[arch] = self.session.host.subtask(method='createrepo',
                                                       arglist=arglist,
                                                       label=arch,
                                                       parent=self.id,
                                                       arch='noarch')
        # gather subtask results
        if subtasks:
            results = self.wait(to_list(subtasks.values()), all=True, failany=True)
            for (arch, task_id) in six.iteritems(subtasks):
                data[arch] = results[task_id]

        # finalize
        kwargs = {}
        if cloned_archs:
            kwargs['repo_json_updates'] = {
                'cloned_from_repo_id': oldrepo['id'],
                'cloned_archs': cloned_archs,
            }
        self.session.host.repoDone(repo_id, data, **kwargs)
        return repo_id, event_id


class CreaterepoTask(BaseTaskHandler):

    Methods = ['createrepo']
    _taskWeight = 1.5

    def handler(self, repo_id, arch, oldrepo):
        # arch is the arch of the repo, not the task
        rinfo = self.session.repoInfo(repo_id, strict=True)
        if rinfo['state'] != koji.REPO_INIT:
            raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % rinfo)
        self.repo_id = rinfo['id']
        self.pathinfo = koji.PathInfo(self.options.topdir)
        toprepodir = self.pathinfo.repo(repo_id, rinfo['tag_name'])
        self.repodir = '%s/%s' % (toprepodir, arch)
        if not os.path.isdir(self.repodir):
            top_repos_dir = joinpath(self.options.topdir, "repos")
            if not os.path.isdir(top_repos_dir):
                # missing or incorrect mount?
                # refuse and let another host try
                raise RefuseTask("No access to repos dir %s" % top_repos_dir)
            else:
                # we seem to have fs access, but dir is missing, perhaps a repo_init bug?
                raise koji.GenericError("Repo directory missing: %s" % self.repodir)
        groupdata = os.path.join(toprepodir, 'groups', 'comps.xml')
        # set up our output dir
        self.outdir = '%s/repo' % self.workdir
        self.datadir = '%s/repodata' % self.outdir
        pkglist = os.path.join(self.repodir, 'pkglist')
        if os.path.getsize(pkglist) == 0:
            pkglist = None
        self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo)
        external_repos = self.session.getExternalRepoList(
            rinfo['tag_id'], event=rinfo['create_event'])
        if external_repos:
            self.merge_repos(external_repos, arch, groupdata)
        elif pkglist is None:
            with open(os.path.join(self.datadir, "EMPTY_REPO"), 'wt') as fo:
                fo.write("This repo is empty because its tag has no content for this arch\n")

        tag = self.session.getTag(rinfo['tag_id'], event=rinfo['create_event'],
                                  strict=True)['name']
        self.run_callbacks('postCreateRepo', tag=tag, repodir=self.outdir,
                           repo_id=self.repo_id, arch=arch)

        uploadpath = self.getUploadDir()
        files = []
        for f in os.listdir(self.datadir):
            files.append(f)
            self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f)
        return [uploadpath, files]

    def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):
        koji.ensuredir(self.outdir)
        if self.options.use_createrepo_c:
            cmd = ['/usr/bin/createrepo_c', '--error-exit-val']
        else:
            cmd = ['/usr/bin/createrepo']
        cmd.extend(['-vd', '-o', self.outdir])
        if pkglist is not None:
            cmd.extend(['-i', pkglist])
        if os.path.isfile(groupdata):
            cmd.extend(['-g', groupdata])
        # attempt to recycle repodata from last repo
        if pkglist and oldrepo and self.options.createrepo_update:
            # old repo could be from inherited tag, so path needs to be
            # composed from that tag, not rinfo['tag_name']
            oldrepo = self.session.repoInfo(oldrepo['id'], strict=True)
            oldpath = self.pathinfo.repo(oldrepo['id'], oldrepo['tag_name'])
            olddatadir = '%s/%s/repodata' % (oldpath, arch)
            if not os.path.isdir(olddatadir):
                self.logger.warning("old repodata is missing: %s" % olddatadir)
            else:
                shutil.copytree(olddatadir, self.datadir)
                oldorigins = os.path.join(self.datadir, 'pkgorigins.gz')
                if os.path.isfile(oldorigins):
                    # remove any previous origins file and rely on mergerepos
                    # to rewrite it (if we have external repos to merge)
                    os.unlink(oldorigins)
                cmd.append('--update')
                if self.options.createrepo_skip_stat:
                    cmd.append('--skip-stat')
        # note: we can't easily use a cachedir because we do not have write
        # permission. The good news is that with --update we won't need to
        # be scanning many rpms.
        if pkglist is None:
            cmd.append(self.outdir)
        else:
            cmd.append(self.repodir)

        logfile = '%s/createrepo.log' % self.workdir
        status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
        if not isSuccess(status):
            raise koji.GenericError('failed to create repo: %s'
                                    % parseStatus(status, format_shell_cmd(cmd)))

    def _get_mergerepo_c_version(self):
        cmd = ['/usr/bin/mergerepo_c', '--version']
        try:
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
            out, _ = proc.communicate()
            status = proc.wait()
            if status != 0:
                self.logger.warning("Unable to detect mergerepo_c version")
                return None
        except Exception:
            self.logger.warning("Unable to detect mergerepo_c version")
            return None
        out = out.decode().strip()
        # Expects output like: "Version: 0.15.11 (Features: DeltaRPM LegacyWeakdeps )"
        m = re.match(r'Version: (\d+).(\d+).(\d+).*', out)
        if not m:
            self.logger.warning("Unable to parse mergerepo_c version")
            return None
        version = m.groups()
        version = [int(x) for x in version]
        return tuple(version)

    def merge_repos(self, external_repos, arch, groupdata):
        # group repos by merge type
        repos_by_mode = {}
        for repo in external_repos:
            repos_by_mode.setdefault(
                repo.get('merge_mode', 'koji'), []).append(repo)

        # figure out merge mode
        if len(repos_by_mode) > 1:
            # TODO: eventually support mixing merge modes
            raise koji.GenericError('Found multiple merge modes for external '
                                    'repos: %s\n' % repos_by_mode.keys())
        merge_mode = to_list(repos_by_mode.keys())[0]

        # move current repo to the premerge location
        localdir = '%s/repo_%s_premerge' % (self.workdir, self.repo_id)
        os.rename(self.outdir, localdir)
        koji.ensuredir(self.outdir)

        # generate repo url list, starting with our local premerge repo
        repos = ['file://' + localdir + '/']
        for repo in external_repos:
            if repo.get('arches') and arch not in repo['arches'].split():
                # ignore external repo with non-relevant archlist
                continue
            ext_url = repo['url']
            # substitute $arch in the url with the arch of the repo we're generating
            ext_url = ext_url.replace('$arch', arch)
            repos.append(ext_url)

        mergerepo_c_version = None
        if self.options.use_createrepo_c or six.PY3:
            mergerepo_c_version = self._get_mergerepo_c_version()

        # construct command
        if merge_mode == 'simple':
            if mergerepo_c_version and mergerepo_c_version >= (0, 13, 0):
                cmd = ['/usr/bin/mergerepo_c', '--koji', '--simple']
            elif six.PY3:
                # koji's mergerepos script only works on python2
                raise koji.GenericError("mergerepo_c is not installed or has low version: "
                                        "%s (0.13.0 needed for --simple)" %
                                        ".".join([str(d) for d in mergerepo_c_version or [None]]))
            else:
                cmd = ['/usr/libexec/kojid/mergerepos',
                       '--mode', 'simple',
                       '--tempdir', self.workdir]
        elif merge_mode == 'bare':
            # "bare" merge mode for repos with modular metadata
            # forces use of mergerepo_c
            cmd = ['/usr/bin/mergerepo_c', '--pkgorigins', '--all']
        elif self.options.use_createrepo_c or six.PY3:
            cmd = ['/usr/bin/mergerepo_c', '--koji']
        else:
            cmd = ['/usr/libexec/kojid/mergerepos', '--tempdir', self.workdir]

        if merge_mode != 'bare':
            blocklist = self.repodir + '/blocklist'
            cmd.extend(['-b', blocklist])
        cmd.extend(['-a', arch, '-o', self.outdir])
        if cmd[0].endswith('mergerepo_c') and mergerepo_c_version \
           and mergerepo_c_version >= (0, 15, 11):
            cmd.append('--arch-expand')
        if os.path.isfile(groupdata):
            cmd.extend(['-g', groupdata])
        for repo in repos:
            cmd.extend(['-r', repo])

        # run command
        logfile = '%s/mergerepos.log' % self.workdir
        env = {'TMPDIR': self.workdir}
        status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(),
                            logerror=True, env=env)
        if not isSuccess(status):
            raise koji.GenericError('failed to merge repos: %s'
                                    % parseStatus(status, format_shell_cmd(cmd)))


class NewDistRepoTask(BaseTaskHandler):
    Methods = ['distRepo']
    _taskWeight = 0.1

    def handler(self, tag, repo_id, keys, task_opts):
        tinfo = self.session.getTag(tag, strict=True, event=task_opts['event'])
        if len(task_opts['arch']) == 0:
            arches = tinfo['arches'] or ''
            task_opts['arch'] = arches.split()
        if len(task_opts['arch']) == 0:
            raise koji.GenericError('No arches specified nor for the tag!')
        subtasks = {}
        # weed out subarchitectures
        canonArches = set()
        for arch in task_opts['arch']:
            canonArches.add(koji.canonArch(arch))
        arch32s = set()
        for arch in canonArches:
            if not koji.arch.isMultiLibArch(arch):
                arch32s.add(arch)
        for arch in arch32s:
            # we do 32-bit multilib arches first so the 64-bit ones can
            # get a task ID and wait for them to complete
            arglist = [tag, repo_id, arch, keys, task_opts]
            subtasks[arch] = self.session.host.subtask(
                method='createdistrepo', arglist=arglist, label=arch,
                parent=self.id, arch='noarch')
        if len(subtasks) > 0 and task_opts['multilib']:
            self.wait(to_list(subtasks.values()), all=True, failany=True)
            for arch in arch32s:
                # move the 32-bit task output to the final resting place
                # so the 64-bit arches can use it for multilib
                upload_dir = koji.pathinfo.taskrelpath(subtasks[arch])
                self.session.host.distRepoMove(repo_id, upload_dir, arch)
        for arch in canonArches:
            # do the other arches
            if arch not in arch32s:
                arglist = [tag, repo_id, arch, keys, task_opts]
                subtasks[arch] = self.session.host.subtask(
                    method='createdistrepo', arglist=arglist, label=arch,
                    parent=self.id, arch='noarch')
        # wait for 64-bit subtasks to finish
        self.wait(to_list(subtasks.values()), all=True, failany=True)
        for (arch, task_id) in six.iteritems(subtasks):
            if task_opts['multilib'] and arch in arch32s:
                # already moved above
                continue
            upload_dir = koji.pathinfo.taskrelpath(subtasks[arch])
            self.session.host.distRepoMove(repo_id, upload_dir, arch)
        self.session.host.repoDone(repo_id, {})
        return 'Dist repository #%s successfully generated' % repo_id


class createDistRepoTask(BaseTaskHandler):
    Methods = ['createdistrepo']
    _taskWeight = 1.5

    archmap = {'s390x': 's390', 'ppc64': 'ppc', 'x86_64': 'i686'}
    compat = {"i386": ("athlon", "i686", "i586", "i486", "i386", "noarch"),
              "x86_64": ("amd64", "ia32e", "x86_64", "noarch"),
              "x86_64_v2": ("amd64", "ia32e", "x86_64", "x86_64_v2", "noarch"),
              "ia64": ("ia64", "noarch"),
              "ppc": ("ppc", "noarch"),
              "ppc64": ("ppc64p7", "ppc64pseries", "ppc64iseries", "ppc64", "noarch"),
              "ppc64le": ("ppc64le", "noarch"),
              "s390": ("s390", "noarch"),
              "s390x": ("s390x", "noarch"),
              "sparc": ("sparcv9v", "sparcv9", "sparcv8", "sparc", "noarch"),
              "sparc64": ("sparc64v", "sparc64", "noarch"),
              "alpha": ("alphaev6", "alphaev56", "alphaev5", "alpha", "noarch"),
              "arm": ("arm", "armv4l", "armv4tl", "armv5tel", "armv5tejl", "armv6l", "armv7l",
                      "noarch"),
              "armhfp": ("armv7hl", "armv7hnl", "noarch"),
              "aarch64": ("aarch64", "noarch"),
              "riscv64": ("riscv64", "noarch"),
              "sw_64": ("sw_64", "noarch"),
              "loongarch64": ("loongarch64", "noarch"),
              "src": ("src",)
              }

    biarch = {"ppc": "ppc64", "x86_64": "i386", "sparc":
              "sparc64", "s390x": "s390", "ppc64": "ppc"}

    def handler(self, tag, repo_id, arch, keys, opts):
        # arch is the arch of the repo, not the task
        self.rinfo = self.session.repoInfo(repo_id, strict=True)
        if self.rinfo['state'] != koji.REPO_INIT:
            raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo)
        groupdata = os.path.join(
            koji.pathinfo.distrepo(repo_id, self.rinfo['tag_name']),
            'groups', 'comps.xml')

        # set up our output dir
        self.repodir = '%s/repo' % self.workdir
        self.repo_files = []
        koji.ensuredir(self.repodir)
        self.subrepos = set()

        # gather oldpkgs data if delta option in use
        oldpkgs = []
        if opts.get('delta'):
            # should be a list of repo ids to delta against
            for delta_repo_id in opts['delta']:
                oldrepo = self.session.repoInfo(delta_repo_id, strict=True)
                if not oldrepo['dist']:
                    raise koji.GenericError("Base repo for deltas must also "
                                            "be a dist repo")
                    # regular repos don't actually have rpms, just pkglist
                path = koji.pathinfo.distrepo(delta_repo_id, oldrepo['tag_name'])
                if not os.path.exists(path):
                    raise koji.GenericError('Base drpm repo missing: %s' % path)
                # note: since we're using the top level dir, this will handle
                #       split repos as well
                oldpkgs.append(path)

        oldrepo = self.session.getRepo(tag, dist=True, state=koji.REPO_READY)
        oldrepodata = None
        if oldrepo:
            oldrepodir = koji.pathinfo.distrepo(oldrepo['id'], tag)

        # sort out our package list(s)
        self.uploadpath = self.getUploadDir()
        self.get_rpms(tag, arch, keys, opts)
        if opts['multilib'] and koji.arch.isMultiLibArch(arch):
            self.do_multilib(arch, self.archmap[arch], opts['multilib'])
        self.split_pkgs(opts)
        self.write_kojipkgs()
        self.write_pkglist()
        self.link_pkgs()

        # generate the repodata
        if oldrepo:
            oldrepodata = os.path.join(oldrepodir, arch, 'repodata')
        self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir,
                           groupdata, oldpkgs=oldpkgs, oldrepodata=oldrepodata,
                           zck=opts.get('zck'), zck_dict_dir=opts.get('zck_dict_dir'),
                           createrepo_skip_stat=opts.get('createrepo_skip_stat'))
        for subrepo in self.subrepos:
            if oldrepo:
                oldrepodata = os.path.join(oldrepodir, arch, subrepo, 'repodata')
            self.do_createrepo(
                '%s/%s' % (self.repodir, subrepo),
                '%s/%s/pkglist' % (self.repodir, subrepo),
                groupdata, oldpkgs=oldpkgs, oldrepodata=oldrepodata,
                logname='createrepo_%s' % subrepo,
                zck=opts.get('zck'),
                zck_dict_dir=opts.get('zck_dict_dir'))
        if len(self.kojipkgs) == 0:
            fn = os.path.join(self.repodir, "repodata", "EMPTY_REPO")
            with open(fn, 'wt') as fp:
                fp.write("This repo is empty because its tag has no content "
                         "for this arch\n")

        self.run_callbacks('postCreateDistRepo', tag=tag, repodir=self.repodir,
                           repo_id=repo_id, arch=arch, keys=keys, opts=opts)

        # upload repo files
        self.upload_repo()
        self.upload_repo_manifest()

    def upload_repo_file(self, relpath):
        """Upload a file from the repo

        relpath should be relative to self.repodir
        """

        localpath = '%s/%s' % (self.repodir, relpath)
        reldir = os.path.dirname(relpath)
        if reldir:
            uploadpath = "%s/%s" % (self.uploadpath, reldir)
            fn = os.path.basename(relpath)
        else:
            uploadpath = self.uploadpath
            fn = relpath
        self.session.uploadWrapper(localpath, uploadpath, fn)
        self.repo_files.append(relpath)

    def upload_repo(self):
        """Traverse the repo and upload needed files

        We omit the symlinks we made for the rpms
        """
        for dirpath, dirs, files in os.walk(self.repodir):
            reldir = os.path.relpath(dirpath, self.repodir)
            for filename in files:
                path = "%s/%s" % (dirpath, filename)
                if os.path.islink(path):
                    continue
                relpath = "%s/%s" % (reldir, filename)
                self.upload_repo_file(relpath)

    def upload_repo_manifest(self):
        """Upload a list of the repo files we've uploaded"""
        fn = '%s/repo_manifest' % self.workdir
        koji.dump_json(fn, self.repo_files, indent=4)
        self.session.uploadWrapper(fn, self.uploadpath)

    def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None,
                      logname=None, oldrepodata=None, zck=False, zck_dict_dir=None,
                      createrepo_skip_stat=None):
        """Run createrepo

        This is derived from CreaterepoTask.create_local_repo, but adapted to
        our requirements here

        :param bool|None createrepo_skip_stat: Override default set in kojid.conf. Note, that
                                               in True variant could resulting repo contain
                                               unexpected rpms.


        """
        koji.ensuredir(repodir)
        if self.options.use_createrepo_c:
            cmd = ['/usr/bin/createrepo_c', '--error-exit-val']
        else:
            cmd = ['/usr/bin/createrepo']
            if zck:
                raise koji.GenericError("createrepo doesn't support zchunks")
        if zck_dict_dir and not zck:
            raise koji.GenericError("--zck-dict-dir makes no sense without --zck")
        cmd.extend(['-vd', '-i', pkglist])
        if groupdata and os.path.isfile(groupdata):
            cmd.extend(['-g', groupdata])
        if pkglist and oldrepodata and self.options.createrepo_update:
            if not os.path.isdir(oldrepodata):
                self.logger.warning("old repodata is missing: %s" % oldrepodata)
            else:
                datadir = os.path.join(repodir, 'repodata')
                shutil.copytree(oldrepodata, datadir)
                oldorigins = os.path.join(datadir, 'pkgorigins.gz')
                if os.path.isfile(oldorigins):
                    # remove any previous origins file and rely on mergerepos
                    # to rewrite it (if we have external repos to merge)
                    os.unlink(oldorigins)
                cmd.append('--update')
                if createrepo_skip_stat is not None:
                    skip_stat = createrepo_skip_stat
                else:
                    skip_stat = self.options.distrepo_skip_stat
                if skip_stat:
                    cmd.append('--skip-stat')
        if oldpkgs:
            # generate delta-rpms
            cmd.append('--deltas')
            for op_dir in oldpkgs:
                cmd.extend(['--oldpackagedirs', op_dir])
        if zck:
            cmd.append('--zck')
        if zck_dict_dir:
            zck_dict_dir = os.path.normpath(zck_dict_dir)
            if os.path.isfile(zck_dict_dir):
                raise koji.GenericError("zchunk dir path is file: %s" % zck_dict_dir)
            if not os.path.isdir(zck_dict_dir):
                raise koji.GenericError("zchunk dir path doesn't exist: %s" % zck_dict_dir)
            cmd.extend(['--zck-dict-dir', zck_dict_dir])
        cmd.append(repodir)

        if logname is None:
            logname = 'createrepo'
        logfile = '%s/%s.log' % (self.workdir, logname)
        status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
        if not isSuccess(status):
            raise koji.GenericError('failed to create repo: %s'
                                    % parseStatus(status, format_shell_cmd(cmd)))

    def do_multilib(self, arch, ml_arch, conf):
        repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])
        mldir = os.path.join(repodir, koji.canonArch(ml_arch))
        ml_true = set()  # multilib packages we need to include before depsolve
        ml_conf = os.path.join(koji.pathinfo.work(), conf)

        # read pkgs data from multilib repo
        ml_pkgfile = os.path.join(mldir, 'kojipkgs')
        ml_pkgs = koji.load_json(ml_pkgfile)

        # step 1: figure out which packages are multilib (should already exist)
        dnfbase = dnf.Base()
        mlm = multilib.DevelMultilibMethod(ml_conf)
        fs_missing = set()
        for bnp in self.kojipkgs:
            rpminfo = self.kojipkgs[bnp]
            ppath = rpminfo['_pkgpath']
            dnfbase.fill_sack(load_system_repo=False, load_available_repos=False)
            po = dnfbase.sack.add_cmdline_package(ppath)
            if mlm.select(po):
                # we need a multilib package to be included
                ml_bnp = bnp.replace(arch, self.archmap[arch])
                ml_path = os.path.join(mldir, ml_bnp[0].lower(), ml_bnp)
                # ^ XXX - should actually generate this
                if ml_bnp not in ml_pkgs:
                    # not in our multilib repo
                    self.logger.error('%s (multilib) is not on the filesystem' % ml_path)
                    fs_missing.add(ml_path)
                    # we defer failure so can report all the missing deps
                    continue
                ml_true.add(ml_path)

        # step 2: set up architectures for dnf configuration
        self.logger.info("Resolving multilib for %s using method devel" % arch)
        dnfdir = os.path.join(self.workdir, 'dnf')
        # TODO: unwind this arch mess
        archlist = (arch, 'noarch')
        transaction_arch = arch
        archlist = archlist + self.compat[self.biarch[arch]]
        best_compat = self.compat[self.biarch[arch]][0]
        if koji.arch.archDifference(best_compat, arch) > 0:
            transaction_arch = best_compat
        dnfconfig = """
[main]
debuglevel=2
#pkgpolicy=newest
#exactarch=1
gpgcheck=0
#reposdir=/dev/null
#cachedir=/dnfcache
installroot=%s
#logfile=/dnf.log

[koji-%s]
name=koji multilib task
baseurl=file://%s
enabled=1

""" % (dnfdir, self.id, mldir)
        os.makedirs(os.path.join(dnfdir, "dnfcache"))
        os.makedirs(os.path.join(dnfdir, 'var/lib/rpm'))

        # step 3: proceed with dnf config and set up
        yconfig_path = os.path.join(dnfdir, 'dnf.conf-koji-%s' % arch)
        with koji._open_text_file(yconfig_path, 'wt') as f:
            f.write(dnfconfig)
        self.session.uploadWrapper(yconfig_path, self.uploadpath,
                                   os.path.basename(yconfig_path))
        conf = dnf.conf.Conf()
        conf.reposdir = []  # don't use system repos at all
        conf.read(yconfig_path)
        dnfbase = dnf.Base(conf)
        if hasattr(koji.arch, 'ArchStorage'):
            dnfbase.conf.arch = transaction_arch
        else:
            koji.arch.canonArch = transaction_arch
        dnfbase.read_all_repos()
        dnfbase.fill_sack(load_system_repo=False, load_available_repos=True)
        for pkg in ml_true:
            dnfbase.install(pkg)

        # step 4: execute dnf transaction to get dependencies
        self.logger.info("Resolving dependencies for arch %s" % arch)

        ml_needed = {}
        try:
            dnfbase.resolve()
            self.logger.info('dnf depsolve successfully finished')
            for po in dnfbase.transaction.install_set:
                bnp = os.path.basename(po.localPkg())
                dep_path = os.path.join(mldir, bnp[0].lower(), bnp)
                ml_needed[dep_path] = po
                if not os.path.exists(dep_path):
                    self.logger.error('%s (multilib dep) not on filesystem' % dep_path)
                    fs_missing.add(dep_path)
        except dnf.exceptions.DepsolveError:
            self.logger.error('dnf depsolve was unsuccessful')
            raise

        if len(fs_missing) > 0:
            missing_log = os.path.join(self.workdir, 'missing_multilib.log')
            with koji._open_text_file(missing_log, 'wt') as outfile:
                outfile.write('The following multilib files were missing:\n')
                for ml_path in fs_missing:
                    outfile.write(ml_path + '\n')
            self.session.uploadWrapper(missing_log, self.uploadpath)
            raise koji.GenericError('multilib packages missing. '
                                    'See missing_multilib.log')

        # step 5: update kojipkgs
        for dep_path in ml_needed:
            tspkg = ml_needed[dep_path]
            bnp = os.path.basename(dep_path)
            if bnp in self.kojipkgs:
                # we expect duplication with noarch, but not other arches
                if tspkg.arch != 'noarch':
                    self.logger.warning("Multilib duplicate: %s", bnp)
                continue
            rpminfo = ml_pkgs[bnp].copy()
            # fix _pkgpath, which comes from another task and could be wrong
            # for us
            # TODO: would be better if we could use the proper path here
            rpminfo['_pkgpath'] = dep_path
            rpminfo['_multilib'] = True
            self.kojipkgs[bnp] = rpminfo

    def pick_key(self, keys, avail_keys):
        best = None
        best_idx = None
        for sigkey in avail_keys:
            if sigkey not in keys:
                # skip, not a key we are looking for
                continue
            idx = keys.index(sigkey)
            # lower idx (earlier in list) is more preferrable
            if best is None or best_idx > idx:
                best = sigkey
                best_idx = idx
        return best

    def get_rpms(self, tag_id, arch, keys, opts):
        keys = [key.lower() for key in keys]
        # get the rpm data
        rpms = []
        builddirs = {}
        for a in self.compat[arch]:
            # note: self.compat includes noarch for non-src already
            rpm_iter, builds = self.session.listTaggedRPMS(tag_id,
                                                           event=opts['event'],
                                                           arch=a,
                                                           latest=opts['latest'],
                                                           inherit=opts['inherit'],
                                                           rpmsigs=True)

            for build in builds:
                # disable draft for distRepo so far
                if build.get('draft'):
                    raise koji.BuildError("Draft build: %s is not allowed" % build['nvr'])
                builddirs[build['id']] = koji.pathinfo.build(build)
            rpms += list(rpm_iter)

        # index by id and key
        rpm_idx = {}
        for rpminfo in rpms:
            sigidx = rpm_idx.setdefault(rpminfo['id'], {})
            sigidx[rpminfo['sigkey']] = rpminfo

        # select our rpms
        selected = {}
        for rpm_id in rpm_idx:
            avail_keys = [key.lower() for key in rpm_idx[rpm_id].keys()]
            best_key = self.pick_key(keys, avail_keys)
            if best_key is None:
                # we lack a matching key for this rpm
                fallback = avail_keys[0]
                rpminfo = rpm_idx[rpm_id][fallback].copy()
                rpminfo['sigkey'] = None
                selected[rpm_id] = rpminfo
            else:
                selected[rpm_id] = rpm_idx[rpm_id][best_key]
                selected[rpm_id]['best_key'] = best_key

        # write signed rpms
        log_output = ''
        if opts.get('write_signed_rpms'):
            results = []
            rpm_with_key = []
            with self.session.multicall(batch=1000) as m:
                for rpm_id in selected:
                    if selected[rpm_id].get('best_key'):
                        results.append(m.host.writeSignedRPM(rpm_id, selected[rpm_id]['best_key']))
                        rpm_with_key.append(rpm_id)

            for rpm_id, r in zip(rpm_with_key, results):
                if isinstance(r._result, list):
                    log_output += 'Signed RPM %s is written with %s key.\n' \
                                  % (rpm_id, selected[rpm_id]['best_key'])
                else:
                    log_output += 'FAILED: Signed RPM %s is not written with %s key, ' \
                                  'error: %s\n' % (rpm_id, selected[rpm_id]['best_key'],
                                                   r._result['faultString'])

        if log_output:
            written_log = os.path.join(self.workdir, 'written_signed_rpms.log')
            with koji._open_text_file(written_log, 'at') as outfile:
                outfile.write(log_output)
            self.session.uploadWrapper(written_log, self.uploadpath)

        # generate kojipkgs data and note missing files
        fs_missing = []
        sig_missing = []
        kojipkgs = {}
        for rpm_id in selected:
            rpminfo = selected[rpm_id]
            if rpminfo['sigkey'] is None:
                sig_missing.append(rpm_id)
                if opts['skip_missing_signatures']:
                    continue
                # use the primary copy, if allowed (checked below)
                pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']],
                                     koji.pathinfo.rpm(rpminfo))
            else:
                # use the signed copy
                pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']],
                                     koji.pathinfo.signed(rpminfo, rpminfo['sigkey']))
            if not os.path.exists(pkgpath):
                fs_missing.append(pkgpath)
                # we'll raise an error below
            else:
                bnp = os.path.basename(pkgpath)
                rpminfo['_pkgpath'] = pkgpath
                kojipkgs[bnp] = rpminfo
        self.kojipkgs = kojipkgs

        # report problems
        if len(fs_missing) > 0:
            missing_log = os.path.join(self.workdir, 'missing_files.log')
            with koji._open_text_file(missing_log, 'wt') as outfile:
                outfile.write('Some rpm files were missing.\n'
                              'Most likely, you want to create these signed copies.\n\n'
                              'Missing files:\n')
                for pkgpath in sorted(fs_missing):
                    outfile.write(pkgpath)
                    outfile.write('\n')
            self.session.uploadWrapper(missing_log, self.uploadpath)
            raise koji.GenericError('Packages missing from the filesystem. '
                                    'See missing_files.log.')
        if sig_missing:
            # log missing signatures and possibly error
            missing_log = os.path.join(self.workdir, 'missing_signatures.log')
            with koji._open_text_file(missing_log, 'wt') as outfile:
                outfile.write('Some rpms were missing requested signatures.\n')
                if opts['skip_missing_signatures']:
                    outfile.write('The skip_missing_signatures option was specified, so '
                                  'these files were excluded.\n')
                outfile.write('Requested keys: %r\n\n' % keys)
                outfile.write('# RPM name: available keys\n')
                fmt = '%(name)s-%(version)s-%(release)s.%(arch)s'
                filenames = [[fmt % selected[r], r] for r in sig_missing]
                for fname, rpm_id in sorted(filenames):
                    avail = to_list(rpm_idx.get(rpm_id, {}).keys())
                    outfile.write('%s: %r\n' % (fname, avail))
            self.session.uploadWrapper(missing_log, self.uploadpath)
            if (not opts['skip_missing_signatures'] and
                    not opts['allow_missing_signatures']):
                raise koji.GenericError('Unsigned packages found. See '
                                        'missing_signatures.log')

    def link_pkgs(self):
        for bnp in self.kojipkgs:
            bnplet = bnp[0].lower()
            ddir = os.path.join(self.repodir, 'Packages', bnplet)
            koji.ensuredir(ddir)
            dst = os.path.join(ddir, bnp)
            pkgpath = self.kojipkgs[bnp]['_pkgpath']
            self.logger.debug("os.symlink(%r, %r(", pkgpath, dst)
            os.symlink(pkgpath, dst)

    def split_pkgs(self, opts):
        '''Direct rpms to subrepos if needed'''
        for rpminfo in self.kojipkgs.values():
            if opts.get('split_debuginfo') and koji.is_debuginfo(rpminfo['name']):
                rpminfo['_subrepo'] = 'debug'
                self.subrepos.add('debug')

    def write_pkglist(self):
        pkgs = []
        subrepo_pkgs = {}
        for bnp in self.kojipkgs:
            rpminfo = self.kojipkgs[bnp]
            bnplet = bnp[0].lower()
            subrepo = rpminfo.get('_subrepo')
            if subrepo:
                # note the ../
                subrepo_pkgs.setdefault(subrepo, []).append(
                    '../Packages/%s/%s\n' % (bnplet, bnp))
            else:
                pkgs.append('Packages/%s/%s\n' % (bnplet, bnp))

        with koji._open_text_file('%s/pkglist' % self.repodir, 'wt') as fo:
            for line in pkgs:
                fo.write(line)
        for subrepo in subrepo_pkgs:
            koji.ensuredir('%s/%s' % (self.repodir, subrepo))
            with koji._open_text_file('%s/%s/pkglist' % (self.repodir, subrepo), 'wt') as fo:
                for line in subrepo_pkgs[subrepo]:
                    fo.write(line)

    def write_kojipkgs(self):
        filename = os.path.join(self.repodir, 'kojipkgs')
        koji.dump_json(filename, self.kojipkgs, sort_keys=False)


def get_options():
    """process options from command line and config file"""
    # parse command line args
    logger = logging.getLogger("koji.build")
    parser = OptionParser()
    parser.add_option("-c", "--config", dest="configFile",
                      help="use alternate configuration file", metavar="FILE",
                      default="/etc/kojid/kojid.conf")
    parser.add_option("--user", help="specify user")
    parser.add_option("--password", help="specify password")
    parser.add_option("-f", "--fg", dest="daemon",
                      action="store_false", default=True,
                      help="run in foreground")
    parser.add_option("--single", action="store_true",
                      help="run a single task in foreground, for debugging")
    parser.add_option("--force-lock", action="store_true", default=False,
                      help="force lock for exclusive session")
    parser.add_option("-v", "--verbose", action="store_true", default=False,
                      help="show verbose output")
    parser.add_option("-d", "--debug", action="store_true", default=False,
                      help="show debug output")
    parser.add_option("--debug-task", action="store_true", default=False,
                      help="enable debug output for tasks")
    parser.add_option("--debug-xmlrpc", action="store_true", default=False,
                      help="show xmlrpc debug output")
    parser.add_option("--debug-mock", action="store_true", default=False,
                      # obsolete option
                      help=SUPPRESS_HELP)
    parser.add_option("--skip-main", action="store_true", default=False,
                      help="don't actually run main")
    parser.add_option("--maxjobs", type='int', help="Specify maxjobs")
    parser.add_option("--minspace", type='int', help="Specify minspace")
    parser.add_option("--sleeptime", type='int', help="Specify the polling interval")
    parser.add_option("--admin-emails", type='str', action="store", metavar="EMAILS",
                      help="Comma-separated addresses to send error notices to.")
    parser.add_option("--topdir", help="Specify topdir")
    parser.add_option("--topurl", help="Specify topurl")
    parser.add_option("--workdir", help="Specify workdir")
    parser.add_option("--chroot-tmpdir", help="Specify tmpdir in buildroot")
    parser.add_option("--pluginpath", help="Specify plugin search path")
    parser.add_option("--plugin", action="append", help="Load specified plugin")
    parser.add_option("--mockdir", help="Specify mockdir")
    parser.add_option("--mockuser", help="User to run mock as")
    parser.add_option("-s", "--server", help="url of XMLRPC server")
    parser.add_option("--pkgurl", help=SUPPRESS_HELP)
    (options, args) = parser.parse_args()

    if args:
        parser.error("incorrect number of arguments")
        # not reached
        assert False  # pragma: no cover

    # --single implies --fg
    if options.single:
        options.daemon = False

    # load local config
    config = koji.read_config_files(options.configFile, raw=True)
    for x in config.sections():
        if x != 'kojid':
            quit('invalid section found in config file: %s' % x)
    defaults = {'sleeptime': 15,
                'maxjobs': 10,
                'buildroot_basic_cleanup_delay': 120,
                'buildroot_final_cleanup_delay': 86400,
                'literal_task_arches': '',
                'minspace': 8192,
                'admin_emails': None,
                'log_level': None,
                'topdir': '/mnt/koji',
                'topurl': None,
                'workdir': '/var/tmp/koji',
                'chroot_tmpdir': '/chroot_tmpdir',
                'pluginpath': '/usr/lib/koji-builder-plugins',
                'mockdir': '/var/lib/mock',
                'mockuser': 'kojibuilder',
                'packager': 'Koji',
                'vendor': 'Koji',
                'distribution': 'Koji',
                'mockhost': 'koji-linux-gnu',
                'smtphost': 'example.com',
                'smtp_user': None,
                'smtp_pass': None,
                'from_addr': 'Koji Build System <buildsys@example.com>',
                'krb_principal': None,
                'host_principal_format': 'compile/%s@EXAMPLE.COM',
                'keytab': '/etc/kojid/kojid.keytab',
                'ccache': '/var/tmp/kojid.ccache',
                'server': None,
                'user': None,
                'password': None,
                'retry_interval': 60,
                'max_retries': 120,
                'offline_retry': True,
                'offline_retry_interval': 120,
                'log_timestamps': False,
                'timeout': None,
                'no_ssl_verify': False,
                'use_fast_upload': True,
                'use_createrepo_c': True,
                'createrepo_skip_stat': True,
                'createrepo_update': True,
                'distrepo_skip_stat': False,
                'copy_old_repodata': False,
                'mock_bootstrap_image': False,
                'pkgurl': None,
                'allowed_scms': '',
                'allowed_scms_use_config': True,
                'allowed_scms_use_policy': False,
                'scm_credentials_dir': None,
                'support_rpm_source_layout': True,
                'yum_proxy': None,
                'maven_repo_ignore': '*.md5 *.sha1 maven-metadata*.xml _maven.repositories '
                                     '_remote.repositories resolver-status.properties '
                                     '*.lastUpdated',
                'failed_buildroot_lifetime': 3600 * 4,
                'rpmbuild_timeout': 3600 * 24,
                'oz_install_timeout': 0,
                'xz_options': '-z6T0',
                'task_avail_delay': 300,
                'cert': None,
                'serverca': None,
                'allow_noverifyssl': False,
                'allow_password_in_scm_url': False}
    if config.has_section('kojid'):
        for name, value in config.items('kojid'):
            if name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval',
                        'max_retries', 'offline_retry_interval', 'failed_buildroot_lifetime',
                        'timeout', 'rpmbuild_timeout', 'oz_install_timeout',
                        'task_avail_delay', 'buildroot_basic_cleanup_delay',
                        'buildroot_final_cleanup_delay']:
                try:
                    defaults[name] = int(value)
                except ValueError:
                    quit("value for %s option must be a valid integer" % name)
            elif name in ['offline_retry', 'use_createrepo_c', 'createrepo_skip_stat',
                          'createrepo_update', 'use_fast_upload', 'support_rpm_source_layout',
                          'build_arch_can_fail', 'no_ssl_verify', 'log_timestamps',
                          'allow_noverifyssl', 'allowed_scms_use_config',
                          'allowed_scms_use_policy', 'allow_password_in_scm_url',
                          'distrepo_skip_stat', 'copy_old_repodata']:
                defaults[name] = config.getboolean('kojid', name)
            elif name in ['plugin', 'plugins']:
                defaults['plugin'] = value.split()
            elif name in to_list(defaults.keys()):
                defaults[name] = value
            elif name.upper().startswith('RLIMIT_'):
                defaults[name.upper()] = value
            else:
                quit("unknown config option: %s" % name)
    for name, value in defaults.items():
        if getattr(options, name, None) is None:
            setattr(options, name, value)

    # honor topdir
    if options.topdir:
        koji.BASEDIR = options.topdir
        koji.pathinfo.topdir = options.topdir

    # make sure workdir exists
    if not os.path.exists(options.workdir):
        koji.ensuredir(options.workdir)

    if not options.server:
        msg = "the server option is required"
        logger.error(msg)
        parser.error(msg)

    if not options.topurl:
        msg = "the topurl option is required"
        logger.error(msg)
        parser.error(msg)

    topurls = options.topurl.split()
    options.topurls = topurls
    if len(topurls) > 1:
        # XXX - fix the rest of the code so this is not necessary
        options.topurl = topurls[0]

    if options.pkgurl:
        logger.warning("The pkgurl option is obsolete")
    if options.debug_mock:
        logger.warning("The debug-mock option is obsolete")

    # special handling for cert defaults
    cert_defaults = {
        'cert': '/etc/kojid/client.crt',
        'serverca': '/etc/kojid/serverca.crt',
    }
    for name in cert_defaults:
        if getattr(options, name, None) is None:
            fn = cert_defaults[name]
            if os.path.exists(fn):
                setattr(options, name, fn)

    return options


def quit(msg=None, code=1):
    if msg:
        logging.getLogger("koji.build").error(msg)
        sys.stderr.write('%s\n' % msg)
        sys.stderr.flush()
    sys.exit(code)


if __name__ == "__main__":
    koji.add_file_logger("koji", "/var/log/kojid.log")
    # note we're setting logging params for all of koji*
    options = get_options()
    if options.log_level:
        lvl = getattr(logging, options.log_level, None)
        if lvl is None:
            quit("Invalid log level: %s" % options.log_level)
        logging.getLogger("koji").setLevel(lvl)
    else:
        logging.getLogger("koji").setLevel(logging.WARN)
    if options.debug:
        logging.getLogger("koji").setLevel(logging.DEBUG)
    elif options.verbose:
        logging.getLogger("koji").setLevel(logging.INFO)
    if options.debug_task:
        logging.getLogger("koji.build.BaseTaskHandler").setLevel(logging.DEBUG)
    if options.admin_emails:
        koji.add_mail_logger("koji", options.admin_emails)

    # start a session and login
    session_opts = koji.grab_session_options(options)
    glob_session = koji.ClientSession(options.server, session_opts)
    if options.cert and os.path.isfile(options.cert):
        try:
            # authenticate using SSL client certificates
            glob_session.ssl_login(options.cert, None, options.serverca)
        except koji.AuthError as e:
            quit("Error: Unable to log in: %s" % e)
        except requests.exceptions.ConnectionError:
            quit("Error: Unable to connect to server %s" % (options.server))
    elif options.user:
        try:
            # authenticate using user/password
            glob_session.login()
        except koji.AuthError:
            quit("Error: Unable to log in. Bad credentials?")
        except requests.exceptions.ConnectionError:
            quit("Error: Unable to connect to server %s" % (options.server))
    elif reqgssapi:
        krb_principal = options.krb_principal
        if krb_principal is None:
            krb_principal = options.host_principal_format % socket.getfqdn()
        try:
            # Check ccache is not empty or authentication will fail
            if os.path.exists(options.ccache) and os.stat(options.ccache).st_size == 0:
                os.remove(options.ccache)

            glob_session.gssapi_login(principal=krb_principal,
                                      keytab=options.keytab,
                                      ccache=options.ccache)
        except Krb5Error as e:
            quit("Kerberos authentication failed: %s" % e.args)
        except socket.error as e:
            quit("Could not connect to Kerberos authentication service: '%s'" % e.args[1])
    else:
        quit("No username/password/certificate supplied and Kerberos missing or not configured")
    # make session exclusive
    try:
        glob_session.exclusiveSession(force=options.force_lock)
    except koji.AuthLockError:
        quit("Error: Unable to get lock. Trying using --force-lock")
    if not glob_session.logged_in:
        quit("Error: Unknown login error")
    # make sure it works
    try:
        ret = glob_session.echo("OK")
    except requests.exceptions.ConnectionError:
        quit("Error: Unable to connect to server %s" % (options.server))
    if ret != ["OK"]:
        quit("Error: incorrect server response: %r" % (ret))

    # run main
    if options.daemon:
        # detach
        koji.daemonize()
        main(options, glob_session)
        # not reached
        assert False  # pragma: no cover
    elif not options.skip_main:
        koji.add_stderr_logger("koji")
        main(options, glob_session)