#!/usr/bin/python

# Koji build daemon
# Copyright (c) 2005-2007 Red Hat
#
#    Koji is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; 
#    version 2.1 of the License.
#
#    This software is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this software; if not, write to the Free Software
#    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
#
# Authors:
#       Mike McLean <mikem@redhat.com>

try:
    import krbV
except ImportError:
    pass
import base64
import koji
import koji.plugin
import koji.util
import commands
import errno
import glob
import logging
import logging.handlers
from koji.util import md5_constructor
import os
import pprint
import pwd
import grp
import random
import re
import rpm
import shutil
import signal
import smtplib
import socket
import sys
import time
import datetime
import traceback
import urllib2
import urlparse
import xmlrpclib
from ConfigParser import ConfigParser
from fnmatch import fnmatch
from gzip import GzipFile
from optparse import OptionParser
from StringIO import StringIO
from xmlrpclib import Fault
import pykickstart.parser as ksparser
import pykickstart.handlers.control as kscontrol
import hashlib
import iso9660 # from pycdio

# our private modules
sys.path.insert(0, '/usr/share/koji-builder/lib')
import tasks

class ServerExit(Exception):
    """Raised to shutdown the server"""
    pass

def main():
    global session
    global options
    logger = logging.getLogger("koji.build")
    logger.info('Starting up')
    tm = TaskManager()
    if options.plugin:
        #load plugins
        pt = koji.plugin.PluginTracker(path=options.pluginpath.split(':'))
        for name in options.plugin:
            logger.info('Loading plugin: %s' % name)
            tm.scanPlugin(pt.load(name))
    def shutdown(*args):
        raise SystemExit
    signal.signal(signal.SIGTERM,shutdown)
    taken = False
    while 1:
        try:
            tm.updateBuildroots()
            tm.updateTasks()
            taken = tm.getNextTask()
        except (SystemExit,ServerExit,KeyboardInterrupt):
            logger.warn("Exiting")
            break
        except koji.AuthExpired:
            logger.error('Session expired')
            break
        except koji.RetryError:
            raise
        except:
            # XXX - this is a little extreme
            # log the exception and continue
            logger.error(''.join(traceback.format_exception(*sys.exc_info())))
        try:
            if not taken:
                # Only sleep if we didn't take a task, otherwise retry immediately.
                # The load-balancing code in getNextTask() will prevent a single builder
                # from getting overloaded.
                time.sleep(options.sleeptime)
        except (SystemExit,KeyboardInterrupt):
            logger.warn("Exiting")
            break
    logger.warn("Shutting down, please wait...")
    tm.shutdown()
    session.logout()
    sys.exit(0)

def log_output(path, args, outfile, uploadpath, cwd=None, logerror=0, append=0, chroot=None, env=None):
    """Run command with output redirected.  If chroot is not None, chroot to the directory specified
    before running the command."""
    pid = os.fork()
    if not pid:
        session._forget()
        try:
            if chroot:
                os.chroot(chroot)
            if cwd:
                os.chdir(cwd)
            flags = os.O_CREAT | os.O_WRONLY
            if append:
                flags |= os.O_APPEND
            fd = os.open(outfile, flags, 0666)
            os.dup2(fd, 1)
            if logerror:
                os.dup2(fd, 2)
            # echo the command we're running into the logfile
            os.write(fd, '$ %s\n' % ' '.join(args))
            environ = os.environ.copy()
            if env:
                environ.update(env)
            os.execvpe(path, args, environ)
        except:
            msg = ''.join(traceback.format_exception(*sys.exc_info()))
            if fd:
                try:
                    os.write(fd, msg)
                    os.close(fd)
                except:
                    pass
            print msg
            os._exit(1)
    else:
        if chroot:
            outfile = os.path.normpath(chroot + outfile)
        outfd = None
        remotename = os.path.basename(outfile)
        while True:
            status = os.waitpid(pid, os.WNOHANG)
            time.sleep(1)

            if not outfd:
                try:
                    outfd = file(outfile, 'r')
                except IOError:
                    # will happen if the forked process has not created the logfile yet
                    continue
                except:
                    print 'Error reading log file: %s' % outfile
                    print ''.join(traceback.format_exception(*sys.exc_info()))

            incrementalUpload(remotename, outfd, uploadpath)

            if status[0] != 0:
                if outfd:
                    outfd.close()
                return status[1]

def safe_rmtree(path, unmount=False, strict=True):
    logger = logging.getLogger("koji.build")
    #safe remove: with -xdev the find cmd will not cross filesystems
    #             (though it will cross bind mounts from the same filesystem)
    if not os.path.exists(path):
        logger.debug("No such path: %s" % path)
        return
    if unmount:
        umount_all(path)
    #first rm -f non-directories
    logger.debug('Scrubbing files in %s' % path)
    rv = os.system("find '%s' -xdev \\! -type d -print0 |xargs -0 rm -f" % path)
    msg = 'file removal failed (code %r) for %s' % (rv,path)
    if rv != 0:
        logger.warn(msg)
        if strict:
            raise koji.GenericError, msg
        else:
            return rv
    #them rmdir directories
    #with -depth, we start at the bottom and work up
    logger.debug('Scrubbing directories in %s' % path)
    rv = os.system("find '%s' -xdev -depth -type d -print0 |xargs -0 rmdir" % path)
    msg = 'dir removal failed (code %r) for %s' % (rv,path)
    if rv != 0:
        logger.warn(msg)
        if strict:
            raise koji.GenericError, msg
    return rv

def umount_all(topdir):
    "Unmount every mount under topdir"
    logger = logging.getLogger("koji.build")
    for path in scan_mounts(topdir):
        logger.debug('Unmounting %s' % path)
        cmd = ['umount', '-l', path]
        rv = os.spawnvp(os.P_WAIT,cmd[0],cmd)
        if rv != 0:
            raise koji.GenericError, 'umount failed (exit code %r) for %s' % (rv,path)
    #check mounts again
    remain = scan_mounts(topdir)
    if remain:
        raise koji.GenericError, "Unmounting incomplete: %r" % remain

def scan_mounts(topdir):
    """Search path for mountpoints"""
    mplist = []
    topdir = os.path.normpath(topdir)
    fo = file('/proc/mounts','r')
    for line in fo.readlines():
        path = line.split()[1]
        if path.startswith(topdir):
            mplist.append(path)
    fo.close()
    #reverse sort so deeper dirs come first
    mplist.sort()
    mplist.reverse()
    return mplist

def incrementalUpload(fname, fd, path, retries=5, logger=None):
    if not fd:
        return
    
    while True:
        offset = fd.tell()
        contents = fd.read(65536)
        size = len(contents)
        if size == 0:
            break
        
        data = base64.encodestring(contents)
        digest = md5_constructor(contents).hexdigest()
        del contents
        
        tries = 0
        while True:
            if session.uploadFile(path, fname, size, digest, offset, data):
                break
            
            if tries <= retries:
                tries += 1
                time.sleep(10)
                continue
            else:
                if logger:
                    logger.error("Error uploading file %s to %s at offset %d" % (fname, path, offset))
                else:
                    sys.stderr.write("Error uploading file %s to %s at offset %d\n" % (fname, path, offset))
                break

def _parseStatus(rv, prefix):
    if isinstance(prefix, list) or isinstance(prefix, tuple):
        prefix = ' '.join(prefix)
    if os.WIFSIGNALED(rv):
        return '%s was killed by signal %i' % (prefix, os.WTERMSIG(rv))
    elif os.WIFEXITED(rv):
        return '%s exited with status %i' % (prefix, os.WEXITSTATUS(rv))
    else:
        return '%s terminated for unknown reasons' % prefix

def _isSuccess(rv):
    """Return True if rv indicates successful completion
    (exited with status 0), False otherwise."""
    if os.WIFEXITED(rv) and os.WEXITSTATUS(rv) == 0:
        return True
    else:
        return False

class BuildRoot(object):

    def __init__(self,*args,**kwargs):
        self.logger = logging.getLogger("koji.build.buildroot")
        if len(args) + len(kwargs) == 1:
            # manage an existing mock buildroot
            self._load(*args,**kwargs)
        else:
            self._new(*args,**kwargs)

    def _load(self, data):
        #manage an existing buildroot
        if isinstance(data, dict):
            #assume data already pulled from db
            self.id = data['id']
        else:
            self.id = data
            data = session.getBuildroot(self.id)
        self.task_id = data['task_id']
        self.tag_id = data['tag_id']
        self.tag_name = data['tag_name']
        self.repoid = data['repo_id']
        self.repo_info = session.repoInfo(self.repoid, strict=True)
        self.event_id = self.repo_info['create_event']
        self.br_arch = data['arch']
        self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self)
        self.config = session.getBuildConfig(self.tag_id, event=self.event_id)

    def _new(self, tag, arch, task_id, repo_id=None, install_group='build', setup_dns=False, bind_opts=None):
        """Create a brand new repo"""
        if not repo_id:
            raise koji.BuildrootError, "A repo id must be provided"
        repo_info = session.repoInfo(repo_id, strict=True)
        self.repo_info = repo_info
        self.repoid = self.repo_info['id']
        self.event_id = self.repo_info['create_event']
        self.task_id = task_id
        self.config = session.getBuildConfig(tag, event=self.event_id)
        if not self.config:
            raise koji.BuildrootError("Could not get config info for tag: %s" % tag)
        self.tag_id = self.config['id']
        self.tag_name = self.config['name']
        if self.config['id'] != repo_info['tag_id']:
            raise koji.BuildrootError, "tag/repo mismatch: %s vs %s" \
                    % (self.config['name'], repo_info['tag_name'])
        repo_state = koji.REPO_STATES[repo_info['state']]
        if repo_state == 'EXPIRED':
            # This should be ok. Expired repos are still intact, just not
            # up-to-date (which may be the point in some cases).
            self.logger.info("Requested repo (%i) is no longer current" % repo_id)
        elif repo_state != 'READY':
            raise koji.BuildrootError, "Requested repo (%i) is %s" % (repo_id, repo_state)
        self.br_arch = koji.canonArch(arch)
        self.logger.debug("New buildroot: %(tag_name)s/%(br_arch)s/%(repoid)s" % vars(self))
        id = session.host.newBuildRoot(self.repoid, self.br_arch, task_id=task_id)
        if id is None:
            raise koji.BuildrootError, "failed to get a buildroot id"
        self.id = id
        self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self)
        self.install_group = install_group
        self.setup_dns = setup_dns
        self.bind_opts = bind_opts
        self._writeMockConfig()

    def _writeMockConfig(self):
        global options
        # mock config
        configdir = '/etc/mock/koji'
        configfile = "%s/%s.cfg" % (configdir,self.name)
        self.mockcfg = "koji/%s" % self.name

        opts = {}
        for k in ('repoid', 'tag_name'):
            if hasattr(self, k):
                opts[k] = getattr(self, k)
        for k in ('mockdir', 'topdir', 'topurl', 'packager', 'vendor', 'distribution', 'mockhost'):
            if hasattr(options, k):
                opts[k] = getattr(options, k)
        opts['buildroot_id'] = self.id
        opts['use_host_resolv'] = self.setup_dns
        opts['install_group'] = self.install_group
        opts['bind_opts'] = self.bind_opts
        output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts)

        #write config
        fo = file(configfile,'w')
        fo.write(output)
        fo.close()

    def mock(self, args, skip_setarch=False):
        """Run mock"""
        global options
        mockpath = getattr(options,"mockpath","/usr/bin/mock")
        cmd = [mockpath, "-r", self.mockcfg]
        if options.debug_mock:
            cmd.append('--debug')
        cmd.extend(args)
        self.logger.info(' '.join(cmd))
        pid = os.fork()
        if pid:
            resultdir = self.resultdir()
            uploadpath = self.getUploadPath()
            logs = {}

            finished = False
            while not finished:
                time.sleep(1)
                status = os.waitpid(pid, os.WNOHANG)
                if status[0] != 0:
                    finished = True

                try:
                    results = os.listdir(resultdir)
                except OSError:
                    # will happen when mock hasn't created the resultdir yet
                    continue

                for fname in results:
                    if fname.endswith('.log') and not logs.has_key(fname):
                        logs[fname] = (None, None, 0)

                for (fname, (fd, inode, size)) in logs.items():
                    try:
                        fpath = os.path.join(resultdir, fname)
                        stat_info = os.stat(fpath)
                        if not fd or stat_info.st_ino != inode or stat_info.st_size < size:
                            # either a file we haven't opened before, or mock replaced a file we had open with
                            # a new file and is writing to it, or truncated the file we're reading,
                            # but our fd is pointing to the previous location in the old file
                            if fd:
                                self.logger.info('Rereading %s, inode: %s -> %s, size: %s -> %s' %
                                                 (fpath, inode, stat_info.st_ino, size, stat_info.st_size))
                                fd.close()
                            fd = file(fpath, 'r')
                        logs[fname] = (fd, stat_info.st_ino, stat_info.st_size)
                    except:
                        self.logger.error("Error reading mock log: %s", fpath)
                        self.logger.error(''.join(traceback.format_exception(*sys.exc_info())))
                        continue

                    incrementalUpload(fname, fd, uploadpath, self.logger)
            #clean up and return exit status of command
            for (fname, (fd, inode, size)) in logs.items():
                if fd:
                    fd.close()
            return status[1]

        else:
            #in no case should exceptions propagate past here
            try:
                session._forget()
                if os.getuid() == 0 and hasattr(options,"mockuser"):
                    self.logger.info('Running mock as %s' % options.mockuser)
                    uid,gid = pwd.getpwnam(options.mockuser)[2:4]
                    os.setgroups([grp.getgrnam('mock')[2]])
                    os.setregid(gid,gid)
                    os.setreuid(uid,uid)
                os.execvp(cmd[0],cmd)
            except:
                #diediedie
                print "Failed to exec mock"
                print ''.join(traceback.format_exception(*sys.exc_info()))
                os._exit(1)

    def getUploadPath(self):
        """Get the path that should be used when uploading files to
        the hub."""
        return koji.pathinfo.taskrelpath(self.task_id)

    def uploadDir(self, dirpath, suffix=None):
        """Upload the contents of the given directory to the
        task output directory on the hub.  If suffix is provided,
        append '.' + suffix to the filenames, so that successive uploads
        of the same directory won't overwrite each other, if the files have
        the same name but different contents."""
        if not os.path.isdir(dirpath):
            return
        uploadpath = self.getUploadPath()
        for filename in os.listdir(dirpath):
            filepath = os.path.join(dirpath, filename)
            if os.stat(filepath).st_size > 0:
                if suffix:
                    filename = '%s.%s' % (filename, suffix)
                session.uploadWrapper(filepath, uploadpath, filename)

    def init(self):
        rv = self.mock(['--init'])

        if rv:
            self.expire()
            raise koji.BuildrootError, "could not init mock buildroot, %s" % self._mockResult(rv)
        session.host.setBuildRootList(self.id,self.getPackageList())

    def _mockResult(self, rv):
        if os.WIFEXITED(rv) and os.WEXITSTATUS(rv) == 1:
            logfile = 'build.log'
        else:
            logfile = 'root.log'
        msg = '; see %s for more information' % logfile
        return _parseStatus(rv, 'mock') + msg

    def build_srpm(self, specfile, sourcedir):
        session.host.setBuildRootState(self.id,'BUILDING')
        chroot_sourcedir = sourcedir[len(self.rootdir()):]
        # call "make sources" in the chroot so any required files not stored in
        # the SCM can be retrieved
        args = ['--no-clean', '--unpriv', '--cwd', chroot_sourcedir, '--chroot', 'make', 'sources']

        rv = self.mock(args)

        if rv:
            self.expire()
            raise koji.BuildError, "error retrieving sources, %s" % self._mockResult(rv)

        args = ['--no-clean', '--buildsrpm', '--spec', specfile, '--sources', sourcedir]

        rv = self.mock(args)

        if rv:
            self.expire()
            raise koji.BuildError, "error building srpm, %s" % self._mockResult(rv)

    def build(self,srpm,arch=None):
        # run build
        session.host.setBuildRootState(self.id,'BUILDING')
        args = ['--no-clean']
        if arch:
            args.extend(['--target', arch])
        args.extend(['--rebuild', srpm])
        rv = self.mock(args)

        session.host.updateBuildRootList(self.id,self.getPackageList())
        if rv:
            self.expire()
            raise koji.BuildError, "error building package (arch %s), %s" % (arch, self._mockResult(rv))

    def getPackageList(self):
        """Return a list of packages from the buildroot

        Each member of the list is a dictionary containing the following fields:
            - name
            - version
            - release
            - epoch
            - arch
            - payloadhash
            - size
            - buildtime
        """
        fields = ('name',
                  'version',
                  'release',
                  'epoch',
                  'arch',
                  'sigmd5',
                  'size',
                  'buildtime')
        rpm.addMacro("_dbpath", "%s/var/lib/rpm" % self.rootdir())
        ret = []
        try:
            ts = rpm.TransactionSet()
            for h in ts.dbMatch():
                pkg = koji.get_header_fields(h,fields)
                #skip our fake packages
                if pkg['name'] == 'buildsys-build':
                    #XXX config
                    continue
                pkg['payloadhash'] = koji.hex_string(pkg['sigmd5'])
                del pkg['sigmd5']
                ret.append(pkg)
        finally:
            rpm.delMacro("_dbpath")
        self.markExternalRPMs(ret)
        return ret

    def markExternalRPMs(self, rpmlist):
        """Check rpms against pkgorigins and add external repo data to the external ones

        Modifies rpmlist in place. No return
        """
        external_repos = session.getExternalRepoList(self.repo_info['tag_id'],
                                    event=self.repo_info['create_event'])
        if not external_repos:
            #nothing to do
            return
        #index external repos by expanded url
        erepo_idx = {}
        for erepo in external_repos:
            # substitute $arch in the url with the arch of the repo we're generating
            ext_url = erepo['url'].replace('$arch', self.br_arch)
            erepo_idx[ext_url] = erepo
        pathinfo = koji.PathInfo(topdir='')
        #XXX - cheap hack to get relative paths
        repodir = pathinfo.repo(self.repo_info['id'], self.repo_info['tag_name'])
        relpath = os.path.join(repodir, self.br_arch, 'repodata', 'pkgorigins.gz')
        opts = dict([(k, getattr(options, k)) for k in 'topurl','topdir'])
        fo = koji.openRemoteFile(relpath, **opts)
        #at this point we know there were external repos at the create event,
        #so there should be an origins file.
        origin_idx = {}
        #GzipFile doesn't play nice with urlopen, so we have the following
        fo2 = GzipFile(fileobj=StringIO(fo.read()), mode='r')
        fo.close()
        for line in fo2:
            parts=line.split(None, 2)
            if len(parts) < 2:
                continue
            #first field is formated by yum as [e:]n-v-r.a
            nvra = "%(name)s-%(version)s-%(release)s.%(arch)s" %  koji.parse_NVRA(parts[0])
            origin_idx[nvra] = parts[1]
        fo2.close()
        # mergerepo starts from a local repo in the task workdir, so internal
        # rpms have an odd-looking origin that we need to look for
        localtail = '/repo_%s_premerge/' % self.repo_info['id']
        for rpm_info in rpmlist:
            key = "%(name)s-%(version)s-%(release)s.%(arch)s" % rpm_info
            # src rpms should not show up in rpmlist so we do not have to
            # worry about fixing the arch for them
            ext_url = origin_idx.get(key)
            if not ext_url:
                raise koji.BuildError, "No origin for %s" % key
            erepo = erepo_idx.get(ext_url)
            if not erepo:
                if ext_url.startswith('file://') and ext_url.endswith(localtail):
                    # internal rpm
                    continue
                raise koji.BuildError, "Unknown origin for %s: %s" % (key, ext_url)
            rpm_info['external_repo'] = erepo
            rpm_info['location'] = erepo['external_repo_id']

    def resultdir(self):
        global options
        return "%s/%s/result" % (options.mockdir, self.name)

    def rootdir(self):
        global options
        return "%s/%s/root" % (options.mockdir, self.name)

    def expire(self):
        session.host.setBuildRootState(self.id,'EXPIRED')


class TaskManager(object):

    def __init__(self):
        self.tasks = {}
        self.pids = {}
        self.subsessions = {}
        self.findHandlers()
        self.status = ''
        self.ready = False
        self.host_id = session.host.getID()
        self.logger = logging.getLogger("koji.build.TaskManager")

    def findHandlers(self):
        """Find and index task handlers"""
        handlers = {}
        for v in globals().values():
            if type(v) == type(BaseTaskHandler) and issubclass(v,BaseTaskHandler):
                for method in v.Methods:
                    handlers[method] = v
        self.handlers = handlers

    def scanPlugin(self, plugin):
        """Find task handlers in a plugin"""
        # XXX - this is a very simple implementation for now.
        #       it should be improved
        for v in vars(plugin).itervalues():
            if type(v) == type(tasks.BaseTaskHandler) and issubclass(v,tasks.BaseTaskHandler):
                for method in v.Methods:
                    self.handlers[method] = v

    def shutdown(self):
        """Attempt to shut down cleanly"""
        for task_id in self.pids.keys():
            self.cleanupTask(task_id)
        session.host.freeTasks(self.tasks.keys())
        session.host.updateHost(task_load=0.0,ready=False)

    def updateBuildroots(self):
        """Handle buildroot cleanup/maintenance

        - examine current buildroots on system
        - compare with db
        - clean up as needed
            - /var/lib/mock
            - /etc/mock/koji
        """
        local_br = self._scanLocalBuildroots()
        #query buildroots in db that are not expired
        states = [ koji.BR_STATES[x] for x in ('INIT','WAITING','BUILDING') ]
        db_br = session.listBuildroots(hostID=self.host_id,state=tuple(states))
        # index by id
        db_br = dict([(row['id'],row) for row in db_br])
        st_expired = koji.BR_STATES['EXPIRED']
        for id, br in db_br.items():
            task_id = br['task_id']
            if task_id is None:
                # not associated with a task
                # this makes no sense now, but may in the future
                self.logger.warn("Expiring taskless buildroot: %(id)i/%(tag_name)s/%(arch)s" % br)
                session.host.setBuildRootState(id,st_expired)
            elif not self.tasks.has_key(task_id):
                #task not running - expire the buildroot
                #TODO - consider recycling hooks here (with strong sanity checks)
                self.logger.info("Expiring buildroot: %(id)i/%(tag_name)s/%(arch)s" % br)
                self.logger.debug("Buildroot task: %r, Current tasks: %r" % (task_id,self.tasks.keys()))
                session.host.setBuildRootState(id,st_expired)
                continue
        # get info on local_only buildroots (most likely expired)
        local_only = [id for id in local_br.iterkeys() if not db_br.has_key(id)]
        if local_only:
            missed_br = session.listBuildroots(buildrootID=tuple(local_only))
            #get all the task info in one call
            tasks = []
            for br in missed_br:
                task_id = br['task_id']
                if task_id:
                    tasks.append(task_id)
            #index
            missed_br = dict([(row['id'],row) for row in missed_br])
            tasks = dict([(row['id'],row) for row in session.getTaskInfo(tasks)])
            for id in local_only:
                # Cleaning options
                #   - wait til later
                #   - "soft" clean (leaving empty root/ dir)
                #   - full removal
                data = local_br[id]
                br = missed_br.get(id)
                if not br:
                    self.logger.warn("%(name)s: not in db" % data)
                    continue
                desc = "%(id)i/%(tag_name)s/%(arch)s" % br
                if not br['retire_ts']:
                    self.logger.warn("%s: no retire timestamp" % desc)
                    continue
                age = time.time() - br['retire_ts']
                self.logger.debug("Expired/stray buildroot: %s" % desc)
                if br and br['task_id']:
                    task = tasks.get(br['task_id'])
                    if not task:
                        self.logger.warn("%s: invalid task %s" % (desc, br['task_id']))
                        continue
                    if (task['state'] == koji.TASK_STATES['FAILED'] and age < 3600 * 4):
                        #XXX - this could be smarter
                        # keep buildroots for failed tasks around for a little while
                        self.logger.debug("Keeping failed buildroot: %s" % desc)
                        continue
                topdir = data['dir']
                rootdir = None
                if topdir:
                    rootdir = "%s/root" % topdir
                    try:
                        st = os.lstat(rootdir)
                    except OSError, e:
                        if e.errno == errno.ENOENT:
                            rootdir = None
                        else:
                            self.logger.warn("%s: %s" % (desc, e))
                            continue
                    else:
                        age = min(age, time.time() - st.st_mtime)
                #note: https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=192153)
                #If rpmlib is installing in this chroot, removing it entirely
                #can lead to a world of hurt.
                #We remove the rootdir contents but leave the rootdir unless it
                #is really old
                if age > 3600*24:
                    #dir untouched for a day
                    self.logger.info("Removing buildroot: %s" % desc)
                    if topdir and safe_rmtree(topdir, unmount=True, strict=False) != 0:
                        continue
                    #also remove the config
                    try:
                        os.unlink(data['cfg'])
                    except OSError, e:
                        self.logger.warn("%s: can't remove config: %s" % (desc, e))
                elif age > 120:
                    if rootdir:
                        try:
                            flist = os.listdir(rootdir)
                        except OSError, e:
                            self.logger.warn("%s: can't list rootdir: %s" % (desc, e))
                            continue
                        if flist:
                            self.logger.info("%s: clearing rootdir" % desc)
                        for fn in flist:
                            safe_rmtree("%s/%s" % (rootdir,fn), unmount=True, strict=False)
                else:
                    self.logger.debug("Recent buildroot: %s: %i seconds" % (desc,age))
        self.logger.debug("Local buildroots: %d" % len(local_br))
        self.logger.debug("Active buildroots: %d" % len(db_br))
        self.logger.debug("Expired/stray buildroots: %d" % len(local_only))

    def _scanLocalBuildroots(self):
        #XXX
        configdir = '/etc/mock/koji'
        buildroots = {}
        for f in os.listdir(configdir):
            if not f.endswith('.cfg'):
                continue
            fn = "%s/%s" % (configdir,f)
            if not os.path.isfile(fn):
                continue
            fo = file(fn,'r')
            id = None
            name = None
            for n in xrange(10):
                # data should be in first few lines
                line = fo.readline()
                if line.startswith('# Koji buildroot id:'):
                    try:
                        id = int(line.split(':')[1])
                    except ValueError,IndexError:
                        continue
                if line.startswith('# Koji buildroot name:'):
                    try:
                        name = line.split(':')[1].strip()
                    except ValueError,IndexError:
                        continue
            if id is None or name is None:
                continue
            # see if there's a dir for the buildroot
            vardir = "/var/lib/mock/%s" % name
            #XXX
            buildroots[id] = {}
            buildroots[id]['name'] = name
            buildroots[id]['cfg'] = fn
            buildroots[id]['dir'] = None
            if os.path.isdir(vardir):
                buildroots[id]['dir'] = vardir
        return buildroots

    def updateTasks(self):
        """Read and process task statuses from server

        The processing we do is:
            1) clean up after tasks that are not longer active:
                * kill off processes
                * retire buildroots
                * remove buildroots
                    - with some possible exceptions
            2) wake waiting tasks if appropriate
        """
        tasks = {}
        stale = []
        task_load = 0.0
        if self.pids:
            self.logger.info("pids: %r" % self.pids)
        for task in session.host.getHostTasks():
            self.logger.info("open task: %r" % task)
            # the tasks returned are those that are open and locked
            # by this host.
            id = task['id']
            if not self.pids.has_key(id):
                #We don't have a process for this
                #Expected to happen after a restart, otherwise this is an error
                stale.append(id)
                continue
            tasks[id] = task
            if task.get('alert',False):
                #wake up the process
                self.logger.info("Waking up task: %r" % task)
                os.kill(self.pids[id],signal.SIGUSR2)
            if not task['waiting']:
                task_load += task['weight']
        self.logger.debug("Task Load: %s" % task_load)
        self.task_load = task_load
        self.tasks = tasks
        self.logger.debug("Current tasks: %r" % self.tasks)
        if len(stale) > 0:
            #A stale task is one which is opened to us, but we know nothing
            #about). This will happen after a daemon restart, for example.
            self.logger.info("freeing stale tasks: %r" % stale)
            session.host.freeTasks(stale)
        for id, pid in self.pids.items():
            if self._waitTask(id, pid):
                # the subprocess handles most everything, we just need to clear things out
                if self.cleanupTask(id, wait=False):
                    del self.pids[id]
                if self.tasks.has_key(id):
                    del self.tasks[id]
        for id, pid in self.pids.items():
            if not tasks.has_key(id):
                # expected to happen when:
                #  - we are in the narrow gap between the time the task
                #    records its result and the time the process actually
                #    exits.
                #  - task is canceled
                #  - task is forcibly reassigned/unassigned
                tinfo = session.getTaskInfo(id)
                if tinfo is None:
                    raise koji.GenericError, "Invalid task %r (pid %r)" % (id,pid)
                elif tinfo['state'] == koji.TASK_STATES['CANCELED']:
                    self.logger.info("Killing canceled task %r (pid %r)" % (id,pid))
                    if self.cleanupTask(id):
                        del self.pids[id]
                elif tinfo['host_id'] != self.host_id:
                    self.logger.info("Killing reassigned task %r (pid %r)" % (id,pid))
                    if self.cleanupTask(id):
                        del self.pids[id]
                else:
                    self.logger.info("Lingering task %r (pid %r)" % (id,pid))

    def getNextTask(self):
        self.ready = self.readyForTask()
        session.host.updateHost(self.task_load,self.ready)
        if not self.ready:
            self.logger.info("Not ready for task")
            return False
        hosts, tasks = session.host.getLoadData()
        self.logger.debug("Load Data:")
        self.logger.debug("  hosts: %r" % hosts)
        self.logger.debug("  tasks: %r" % tasks)
        #now we organize this data into channel-arch bins
        bin_hosts = {}  #hosts indexed by bin
        bins = {}       #bins for this host
        our_avail = None
        for host in hosts:
            host['bins'] = []
            if host['id'] == self.host_id:
                #note: task_load reported by server might differ from what we
                #sent due to precision variation
                our_avail = host['capacity'] - host['task_load']
            for chan in host['channels']:
                for arch in host['arches'].split() + ['noarch']:
                    bin = "%s:%s" % (chan,arch)
                    bin_hosts.setdefault(bin,[]).append(host)
                    if host['id'] == self.host_id:
                        bins[bin] = 1
        self.logger.debug("bins: %r" % bins)
        if our_avail is None:
            self.logger.info("Server did not report this host. Are we disabled?")
            return False
        elif not bins:
            self.logger.info("No bins for this host. Missing channel/arch config?")
            return False
        #sort available capacities for each of our bins
        avail = {}
        for bin in bins.iterkeys():
            avail[bin] = [host['capacity'] - host['task_load'] for host in bin_hosts[bin]]
            avail[bin].sort()
            avail[bin].reverse()
        for task in tasks:
            # note: tasks are in priority order
            self.logger.debug("task: %r" % task)
            if self.tasks.has_key(task['id']):
                # we were running this task, but it apparently has been
                # freed or reassigned. We can't do anything with it until
                # updateTasks notices this and cleans up.
                self.logger.debug("Task %(id)s freed or reassigned", task)
                continue
            if task['state'] == koji.TASK_STATES['ASSIGNED']:
                self.logger.debug("task is assigned")
                if self.host_id == task['host_id']:
                    #assigned to us, we can take it regardless
                    if self.takeTask(task['id']):
                        return True
            elif task['state'] == koji.TASK_STATES['FREE']:
                bin = "%(channel_id)s:%(arch)s" % task
                self.logger.debug("task is free, bin=%r" % bin)
                if not bins.has_key(bin):
                    continue
                #see where our available capacity is compared to other hosts for this bin
                #(note: the hosts in this bin are exactly those that could
                #accept this task)
                bin_avail = avail.get(bin, [0])
                self.logger.debug("available capacities for bin: %r" % bin_avail)
                median = bin_avail[(len(bin_avail)-1)/2]
                self.logger.debug("ours: %.2f, median: %.2f" % (our_avail, median))
                if our_avail < median:
                    self.logger.debug("Skipping - available capacity in lower half")
                    #decline for now and give the upper half a chance
                    return False
                #otherwise, we attempt to open the task
                if self.takeTask(task['id']):
                    return True
            else:
                #should not happen
                raise Exception, "Invalid task state reported by server"
        return False

    def _waitTask(self, task_id, pid=None):
        """Wait (nohang) on the task, return true if finished"""
        if pid is None:
            pid = self.pids.get(task_id)
            if not pid:
                raise koji.GenericError, "No pid for task %i" % task_id
        prefix = "Task %i (pid %i)" % (task_id, pid)
        try:
            (childpid, status) = os.waitpid(pid, os.WNOHANG)
        except OSError, e:
            #check errno
            if e.errno != errno.ECHILD:
                #should not happen
                raise
            #otherwise assume the process is gone
            self.logger.info("%s: %s" % (prefix, e))
            return True
        if childpid != 0:
            self.logger.info(_parseStatus(status, prefix))
            return True
        return False

    def _doKill(self, task_id, pid, cmd, sig, timeout, pause):
        """
        Kill the process with the given process ID.
        Return True if the process is successfully killed in
        the given timeout, False otherwise.
        """
        self.logger.info('Checking "%s" (pid %i, taskID %i)...' % (cmd, pid, task_id))
        execname = cmd.split()[0]
        signaled = False
        t = 0.0
        while True:
            status = self._getStat(pid)
            if status and status[1] == cmd and status[2] != 'Z':
                self.logger.info('%s (pid %i, taskID %i) is running' % (execname, pid, task_id))
            else:
                if signaled:
                    self.logger.info('%s (pid %i, taskID %i) was killed by signal %i' % (execname, pid, task_id, sig))
                else:
                    self.logger.info('%s (pid %i, taskID %i) exited' % (execname, pid, task_id))
                return True

            if t >= timeout:
                self.logger.warn('Failed to kill %s (pid %i, taskID %i) with signal %i' %
                                 (execname, pid, task_id, sig))
                return False

            try:
                os.kill(pid, sig)
            except OSError, e:
                # process probably went away, we'll find out on the next iteration
                self.logger.info('Error sending signal %i to %s (pid %i, taskID %i): %s' %
                                 (sig, execname, pid, task_id, e))
            else:
                signaled = True
                self.logger.info('Sent signal %i to %s (pid %i, taskID %i)' %
                                 (sig, execname, pid, task_id))

            time.sleep(pause)
            t += pause

    def _getStat(self, pid):
        """
        Get the stat info for the given pid.
        Return a list of all the fields in /proc/<pid>/stat.
        The second entry will contain the full command-line instead of
        just the command name.
        If the process does not exist, return None.
        """
        try:
            proc_path = '/proc/%i/stat' % pid
            if not os.path.isfile(proc_path):
                return None
            proc_file = file(proc_path)
            procstats = [not field.isdigit() and field or int(field) for field in proc_file.read().split()]
            proc_file.close()

            cmd_path = '/proc/%i/cmdline' % pid
            if not os.path.isfile(cmd_path):
                return None
            cmd_file = file(cmd_path)
            procstats[1] = cmd_file.read().replace('\0', ' ').strip()
            cmd_file.close()
            if not procstats[1]:
                return None

            return procstats
        except IOError, e:
            # process may have already gone away
            return None

    def _childPIDs(self, pid):
        """Recursively get the children of the process with the given ID.
        Return a list containing the process IDs of the children
        in breadth-first order, without duplicates."""
        statsByPPID = {}
        pidcmd = None
        for procdir in os.listdir('/proc'):
            if not procdir.isdigit():
                continue
            procid = int(procdir)
            procstats = self._getStat(procid)
            if not procstats:
                continue
            statsByPPID.setdefault(procstats[3], []).append(procstats)
            if procid == pid:
                pidcmd = procstats[1]

        pids = []
        if pidcmd:
            # only append the pid if it still exists
            pids.append((pid, pidcmd))

        parents = [pid]
        while parents:
            for ppid in parents[:]:
                for procstats in statsByPPID.get(ppid, []):
                    # get the /proc entries with ppid as their parent, and append their pid to the list,
                    # then recheck for their children
                    # pid is the 0th field, ppid is the 3rd field
                    pids.append((procstats[0], procstats[1]))
                    parents.append(procstats[0])
                parents.remove(ppid)

        return pids

    def _killChildren(self, task_id, children, sig=signal.SIGTERM, timeout=2.0, pause=1.0):
        """
        Kill child processes of the given task, as specified in the children list,
        by sending sig.
        Retry every pause seconds, within timeout.
        Remove successfully killed processes from the "children" list.
        """
        for childpid, cmd in children[::-1]:
            # iterate in reverse order so processes whose children are killed might have
            # a chance to cleanup before they're killed
            if self._doKill(task_id, childpid, cmd, sig, timeout, pause):
                children.remove((childpid, cmd))

    def cleanupTask(self, task_id, wait=True):
        """Clean up after task

        - kill children
        - expire session

        Return True if all children were successfully killed, False otherwise.
        """
        pid = self.pids.get(task_id)
        if not pid:
            raise koji.GenericError, "No pid for task %i" % task_id
        children = self._childPIDs(pid)
        if children:
            # send SIGINT once to let mock mock try to clean up
            self._killChildren(task_id, children, sig=signal.SIGINT, pause=3.0)
        if children:
            self._killChildren(task_id, children)
        if children:
            self._killChildren(task_id, children, sig=signal.SIGKILL, timeout=3.0)

        #expire the task's subsession
        session_id = self.subsessions.get(task_id)
        if session_id:
            self.logger.info("Expiring subsession %i (task %i)" % (session_id, task_id))
            try:
                session.logoutChild(session_id)
                del self.subsessions[task_id]
            except:
                #not much we can do about it
                pass
        if wait:
            return self._waitTask(task_id, pid)
        else:
            # task has already been waited on, and we've cleaned
            # up as much as we can
            return True

    def checkSpace(self):
        """See if we have enough space to accept another job"""
        global options
        br_path = options.mockdir
        if not os.path.exists(br_path):
            self.logger.error("No such directory: %s" % br_path)
            raise IOError, "No such directory: %s" % br_path
        fs_stat = os.statvfs(br_path)
        available = fs_stat.f_bavail * fs_stat.f_bsize
        availableMB = available / 1024 / 1024
        self.logger.debug("disk space available in '%s': %i MB", br_path, availableMB)
        if availableMB < options.minspace:
            self.status = "Insufficient disk space: %i MB, %i MB required" % (availableMB, options.minspace)
            self.logger.warn(self.status)
            return False
        return True

    def readyForTask(self):
        """Determine if the system is ready to accept a new task.

        This function measures the system load and tries to determine
        if there is room to accept a new task."""
        #       key resources to track:
        #               disk_space
        #                       df -P path
        #                       df -iP path ?
        #               memory (meminfo/vmstat)
        #                       vmstat fields 3-6 (also 7-8 for swap)
        #                       http://www.redhat.com/advice/tips/meminfo.html
        #               cpu cycles (vmstat?)
        #                       vmstat fields 13-16 (and others?)
        #       others?:
        #               io (iostat/vmstat)
        #               network (netstat?)
        global options
        hostdata = session.host.getHost()
        self.logger.debug('hostdata: %r' % hostdata)
        if not hostdata['enabled']:
            self.status = "Host is disabled"
            self.logger.info(self.status)
            return False
        if self.task_load > hostdata['capacity']:
            self.status = "Over capacity"
            self.logger.info("Task load (%.2f) exceeds capacity (%.2f)" % (self.task_load, hostdata['capacity']))
            return False
        if len(self.tasks) >= options.maxjobs:
            # This serves as a backup to the capacity check and prevents
            # a tremendous number of low weight jobs from piling up
            self.status = "Full queue"
            self.logger.info(self.status)
            return False
        if not self.checkSpace():
            # checkSpace() does its own logging
            return False
        loadavgs = os.getloadavg()
        # this likely treats HT processors the same as real ones
        # but that's fine, it's a conservative test
        maxload = 4.0 * os.sysconf('SC_NPROCESSORS_ONLN')
        if loadavgs[0] > maxload:
            self.status = "Load average %.2f > %.2f" % (loadavgs[0], maxload)
            self.logger.info(self.status)
            return False
        #XXX - add more checks
        return True

    def takeTask(self,task_id):
        """Attempt to open the specified task

        Returns True if successful, False otherwise
        """
        self.logger.info("Attempting to take task %s" %task_id)
        data = session.host.openTask(task_id)
        if data is None:
            self.logger.warn("Could not open")
            return False
        if not data.has_key('request') or data['request'] is None:
            self.logger.warn("Task '%s' has no request" % task_id)
            return False
        id = data['id']
        request = data['request']
        self.tasks[id] = data
        params, method = xmlrpclib.loads(request)
        if self.handlers.has_key(method):
            handlerClass = self.handlers[method]
        elif self.handlers.has_key('default'):
            handlerClass = self.handlers['default']
        else:
            raise koji.GenericError, "No handler found for method '%s'" % method
        if issubclass(handlerClass, tasks.BaseTaskHandler):
            #new style handler needs session and options passed
            handler = handlerClass(id,method,params,session,options)
        else:
            handler = handlerClass(id,method,params)
        # set weight
        session.host.setTaskWeight(task_id,handler.weight())
        if handler.Foreground:
            self.logger.info("running task in foreground")
            handler.setManager(self)
            self.runTask(handler)
        else:
            pid, session_id = self.forkTask(handler)
            self.pids[id] = pid
            self.subsessions[id] = session_id
        return True

    def forkTask(self,handler):
        global session
        #get the subsession before we fork
        newhub = session.subsession()
        session_id = newhub.sinfo['session-id']
        pid = os.fork()
        if pid:
            newhub._forget()
            return pid, session_id
        #in no circumstance should we return after the fork
        #nor should any exceptions propagate past here
        try:
            session._forget()
            #set process group
            os.setpgrp()
            #use the subsession
            session = newhub
            if hasattr(handler, 'session'):
                handler.session = session
            #set a do-nothing handler for sigusr2
            signal.signal(signal.SIGUSR2,lambda *args: None)
            self.runTask(handler)
        finally:
            #diediedie
            try:
                session.logout()
            finally:
                os._exit(0)

    def runTask(self,handler):
        fail = False
        try:
            response = (handler.run(),)
            # note that we wrap response in a singleton tuple
            response = xmlrpclib.dumps(response, methodresponse=1, allow_none=1)
            self.logger.info("RESPONSE: %r" % response)
        except Fault, fault:
            fail = True
            response = xmlrpclib.dumps(fault)
            tb = ''.join(traceback.format_exception(*sys.exc_info())).replace(r"\n", "\n")
            self.logger.warn("FAULT:\n%s" % tb)
        except (SystemExit,ServerExit,KeyboardInterrupt):
            #we do not trap these
            raise
        except:
            fail = True
            # report exception back to server
            e_class, e = sys.exc_info()[:2]
            faultCode = getattr(e_class,'faultCode',1)
            if issubclass(e_class, koji.GenericError):
                #just pass it through
                tb = str(e)
                self.logger.warn(tb)
            else:
                tb = ''.join(traceback.format_exception(*sys.exc_info()))
                self.logger.warn("TRACEBACK: %s" % tb)
            response = xmlrpclib.dumps(xmlrpclib.Fault(faultCode, tb))

        if fail:
            session.host.failTask(handler.id, response)
        else:
            session.host.closeTask(handler.id, response)


class BaseTaskHandler(object):
    """The base class for task handlers

    Each task handler is a class, a new instance of which is created
    to handle each task.
    """

    # list of methods the class can handle
    Methods = []

    # Options:
    Foreground = False

    def __init__(self, id, method, params, workdir=None):
        global options
        self.id = id   #task id
        if method not in self.Methods:
            raise koji.GenericError, 'method "%s" is not supported' % method
        self.method = method
        # handle named parameters
        self.params,self.opts = koji.decode_args(*params)
        if workdir is None:
            workdir = "%s/%s" % (options.workdir, koji.pathinfo.taskrelpath(id))
        self.workdir = workdir
        self.logger = logging.getLogger("koji.build.BaseTaskHandler")

    def setManager(self,manager):
        """Set the manager attribute

        This is only used for foreground tasks to give them access
        to their task manager.
        """
        if not self.Foreground:
            return
        self.manager = manager

    def handler(self):
        """(abstract) the handler for the task."""
        raise NotImplementedError

    def run(self):
        """Execute the task"""
        self.createWorkdir()
        try:
            return self.handler(*self.params,**self.opts)
        finally:
            self.removeWorkdir()

    _taskWeight = 1.0

    def weight(self):
        """Return the weight of the task.

        This is run by the taskmanager before the task is run to determine
        the weight of the task. The weight is an abstract measure of the
        total load the task places on the system while running.

        A task may set _taskWeight for a constant weight different from 1, or
        override this function for more complicated situations.

        Note that task weight is partially ignored while the task is sleeping.
        """
        return getattr(self,'_taskWeight',1.0)

    def createWorkdir(self):
        if self.workdir is None:
            return
        self.removeWorkdir()
        os.makedirs(self.workdir)

    def removeWorkdir(self):
        if self.workdir is None:
            return
        safe_rmtree(self.workdir, unmount=False, strict=True)
        #os.spawnvp(os.P_WAIT, 'rm', ['rm', '-rf', self.workdir])

    def wait(self, subtasks=None, all=False, failany=False):
        """Wait on subtasks

        subtasks is a list of integers (or an integer). If more than one subtask
        is specified, then the default behavior is to return when any of those
        tasks complete. However, if all is set to True, then it waits for all of
        them to complete.  If all and failany are both set to True, then each
        finished task will be checked for failure, and a failure will cause all
        of the unfinished tasks to be cancelled.

        special values:
            subtasks = None     specify all subtasks

        Implementation notes:
            The build daemon forks all tasks as separate processes. This function
            uses signal.pause to sleep. The main process watches subtasks in
            the database and will send the subprocess corresponding to the
            subtask a SIGUSR2 to wake it up when subtasks complete.
        """
        if isinstance(subtasks,int):
            # allow single integer w/o enclosing list
            subtasks = [subtasks]
        session.host.taskSetWait(self.id,subtasks)
        self.logger.debug("Waiting on %r" % subtasks)
        while True:
            finished, unfinished = session.host.taskWait(self.id)
            if len(unfinished) == 0:
                #all done
                break
            elif len(finished) > 0:
                if all:
                    if failany:
                        failed = False
                        for task in finished:
                            try:
                                result = session.getTaskResult(task)
                            except (koji.GenericError, Fault), task_error:
                                self.logger.info("task %s failed or was canceled" % task)
                                failed = True
                                break
                        if failed:
                            self.logger.info("at least one task failed or was canceled, cancelling unfinished tasks")
                            session.cancelTaskChildren(self.id)
                            # reraise the original error now, rather than waiting for
                            # an error in taskWaitResults()
                            raise task_error
                else:
                    # at least one done
                    break
            # signal handler set by TaskManager.forkTask
            self.logger.debug("Pausing...")
            signal.pause()
            # main process will wake us up with SIGUSR2
            self.logger.debug("...waking up")
        self.logger.debug("Finished waiting")
        return dict(session.host.taskWaitResults(self.id,subtasks))

    def getUploadDir(self):
        return koji.pathinfo.taskrelpath(self.id)

    def uploadFile(self, filename, remoteName=None):
        """Upload the file with the given name to the task output directory
        on the hub."""
        # Only upload files with content
        if os.path.isfile(filename) and os.stat(filename).st_size > 0:
            session.uploadWrapper(filename, self.getUploadDir(), remoteName)

    def genImageManifest(self, image, manifile):
        """
        Using iso9660 from pycdio, get the file manifest of the given image,
        and save it to the text file manifile.
        """
        fd = open(manifile, 'w')
        if not fd:
            raise koji.GenericError, \
                'Unable to open manifest file (%s) for writing!' % manifile
        iso = iso9660.ISO9660.IFS(source=image)
        if not iso.is_open():
            raise koji.GenericError, \
                'Could not open %s as an ISO-9660 image!' % image

        # image metadata
        id = iso.get_application_id()
        if id is not None: fd.write("Application ID: %s\n" % id)
        id = iso.get_preparer_id()
        if id is not None: fd.write("Preparer ID: %s\n" % id)
        id = iso.get_publisher_id()
        if id is not None: fd.write("Publisher ID: %s\n" % id)
        id = iso.get_system_id()
        if id is not None: fd.write("System ID: %s\n" % id)
        id = iso.get_volume_id()
        if id is not None: fd.write("Volume ID: %s\n" % id)
        id = iso.get_volumeset_id()
        if id is not None: fd.write("Volumeset ID: %s\n" % id)

        fd.write('\nSize(bytes)  File Name\n')
        manifest = self.listImageDir(iso, '/')
        for a_file in manifest:
            fd.write(a_file)
        fd.close()
        iso.close()

    def listImageDir(self, iso, path):
        """
        Helper function called recursively by getImageManifest. Returns a
        listing of files/directories at the given path in an iso image obj.
        """
        manifest = []
        file_stats = iso.readdir(path)
        for stat in file_stats:
            filename = stat[0]
            size = stat[2]
            is_dir = stat[4] == 2

            if filename == '..':
                continue
            elif filename == '.':
                # path should always end in a trailing /
                filepath = path
            else:
                filepath = path + filename
                # identify directories with a trailing /
                if is_dir:
                    filepath += '/'

            if is_dir and filename != '.':
                # recurse into subdirectories
                manifest.extend(self.listImageDir(iso, filepath))
            else:
                # output information for the current directory and files
                manifest.append("%-10d   %s\n" % (size, filepath))

        return manifest

    def localPath(self, relpath):
        """Return a local path to a remote file.

        If the file is on an nfs mount, use that, otherwise download a copy"""
        if options.topurl:
            self.logger.debug("Downloading %s", relpath)
            url = "%s/%s" % (options.topurl, relpath)
            fsrc = urllib2.urlopen(url)
            fn = "%s/local/%s" % (self.workdir, relpath)
            os.makedirs(os.path.dirname(fn))
            fdst = file(fn, 'w')
            shutil.copyfileobj(fsrc, fdst)
            fsrc.close()
            fdst.close()
        else:
            fn = "%s/%s" % (options.topdir, relpath)
        return fn

    def subtask(self, method, arglist, **opts):
        return session.host.subtask(method, arglist, self.id, **opts)

    def subtask2(self, __taskopts, __method, *args, **kwargs):
        return session.host.subtask2(self.id, __taskopts, __method, *args, **kwargs)

    def find_arch(self, arch, host, tag):
        """
        For noarch tasks, find a canonical arch that is supported by both the host and tag.
        If the arch is anything other than noarch, return it unmodified.
        """
        if arch != "noarch":
            return arch

        # We need a concrete arch. Pick one that:
        #  a) this host can handle
        #  b) the build tag can support
        #  c) is canonical
        host_arches = host['arches']
        if not host_arches:
            raise koji.BuildError, "No arch list for this host: %s" % host['name']
        tag_arches = tag['arches']
        if not tag_arches:
            raise koji.BuildError, "No arch list for tag: %s" % tag['name']
        # index canonical host arches
        host_arches = set([koji.canonArch(a) for a in host_arches.split()])
        # index canonical tag arches
        tag_arches = set([koji.canonArch(a) for a in tag_arches.split()])
        # find the intersection of host and tag arches
        common_arches = list(host_arches & tag_arches)
        if common_arches:
            # pick one of the common arches randomly
            # need to re-seed the prng or we'll get the same arch every time,
            # because we just forked from a common parent
            random.seed()
            arch = random.choice(common_arches)
            self.logger.info('Valid arches: %s, using: %s' % (' '.join(common_arches), arch))
            return arch
        else:
            # no overlap
            raise koji.BuildError, "host %s (%s) does not support any arches of tag %s (%s)" % \
                (host['name'], ', '.join(host_arches), tag['name'], ', '.join(tag_arches))

class FakeTask(BaseTaskHandler):
    Methods = ['someMethod']
    Foreground = True
    def handler(self, *args):
        self.logger.info("This is a fake task.  Args: " + str(args))
        return 42


class SleepTask(BaseTaskHandler):
    Methods = ['sleep']
    _taskWeight = 0.25
    def handler(self, n):
        self.logger.info("Sleeping for %s seconds" % n)
        time.sleep(n)
        self.logger.info("Finished sleeping")

class ForkTask(BaseTaskHandler):
    Methods = ['fork']
    def handler(self, n=5, m=37):
        for i in xrange(n):
            os.spawnvp(os.P_NOWAIT, 'sleep', ['sleep',str(m)])

class WaitTestTask(BaseTaskHandler):
    Methods = ['waittest']
    _taskWeight = 0.1
    def handler(self,count,seconds=10):
        tasks = []
        for i in xrange(count):
            task_id = session.host.subtask(method='sleep',
                                          arglist=[seconds],
                                          label=str(i),
                                          parent=self.id)
            tasks.append(task_id)
        results = self.wait(all=True)
        self.logger.info(pprint.pformat(results))


class SubtaskTask(BaseTaskHandler):
    Methods = ['subtask']
    _taskWeight = 0.1
    def handler(self,n=4):
        if n > 0:
            task_id = session.host.subtask(method='subtask',
                                          arglist=[n-1],
                                          label='foo',
                                          parent=self.id)
            self.wait(task_id)
        else:
            task_id = session.host.subtask(method='sleep',
                                          arglist=[15],
                                          label='bar',
                                          parent=self.id)
            self.wait(task_id)


class DefaultTask(BaseTaskHandler):
    """Used when no matching method is found"""
    Methods = ['default']
    _taskWeight = 0.1
    def __init__(self, id, method, params, workdir=None):
        self.id = id   #task id
        self.method = method
        self.params = params
        self.workdir = None
        self.opts = {}
    def handler(self,*args,**opts):
        raise koji.GenericError, "Invalid method: %s" % self.method


class ShutdownTask(BaseTaskHandler):
    Methods = ['shutdown']
    _taskWeight = 0.0
    Foreground = True
    def handler(self):
        #note: this is a foreground task
        raise ServerExit


class DependantTask(BaseTaskHandler):

    Methods = ['dependantTask']
    #mostly just waiting on other tasks
    _taskWeight = 0.2

    def handler(self, wait_list, task_list):
        for task in wait_list:
            if not isinstance(task, int) or not session.getTaskInfo(task):
                self.logger.debug("invalid task id %s, removing from wait_list" % task)
                wait_list.remove(task)

        # note, tasks in wait_list are not children of this task so we can't
        # just use self.wait()
        while wait_list:
            for task in wait_list[:]:
                if session.taskFinished(task):
                    info = session.getTaskInfo(task)
                    if info and koji.TASK_STATES[info['state']] in ['CANCELED','FAILED']:
                        raise koji.GenericError, "Dependency %s failed to complete." % info['id']
                    wait_list.remove(task)
            # let the system rest before polling again
            time.sleep(1)

        subtasks = []
        for task in task_list:
            # **((len(task)>2 and task[2]) or {}) expands task[2] into opts if it exists, allows for things like 'priority=15'
            task_id = session.host.subtask(method=task[0], arglist=task[1], parent=self.id, **((len(task)>2 and task[2]) or {}))
            if task_id:
                subtasks.append(task_id)
        if subtasks:
            self.wait(subtasks, all=True)

class ChainBuildTask(BaseTaskHandler):

    Methods = ['chainbuild']
    #mostly just waiting on other tasks
    _taskWeight = 0.1

    def handler(self, srcs, target, opts=None):
        if opts.get('scratch'):
            raise koji.BuildError, "--scratch is not allowed with chain-builds"
        target_info = session.getBuildTarget(target)
        if not target_info:
            raise koji.GenericError, 'unknown build target: %s' % target
        for build_level in srcs:
            subtasks = []
            build_tasks = []
            nvrs = []
            for src in build_level:
                if SCM.is_scm_url(src):
                    task_id = session.host.subtask(method='build',
                                                   arglist=[src, target, opts],
                                                   label=src,
                                                   parent=self.id)
                    build_tasks.append(task_id)
                    subtasks.append(task_id)
                else:
                    nvrs.append(src)
            if nvrs:
                task_id = session.host.subtask(method='waitrepo',
                                               arglist=[target_info['build_tag_name'], None, nvrs],
                                               label=','.join(nvrs),
                                               parent=self.id)
                subtasks.append(task_id)
            if not subtasks:
                continue
            self.wait(subtasks, all=True, failany=True)
            if srcs[-1] == build_level:
                continue
            nvrs = []
            for build_task in build_tasks:
                builds = session.listBuilds(taskID=build_task)
                if builds:
                    nvrs.append(builds[0]['nvr'])
            if nvrs:
                task_id = session.host.subtask(method='waitrepo',
                                               arglist=[target_info['build_tag_name'], None, nvrs],
                                               label=','.join(nvrs),
                                               parent=self.id)
                self.wait(task_id, all=True, failany=True)

class BuildTask(BaseTaskHandler):

    Methods = ['build']
    #we mostly just wait on other tasks
    _taskWeight = 0.2

    def handler(self, src, target, opts=None):
        """Handler for the master build task"""
        if opts is None:
            opts = {}
        self.opts = opts
        if opts.get('arch_override') and not opts.get('scratch'):
            raise koji.BuildError, "arch_override is only allowed for scratch builds"
        if opts.get('repo_id') is not None:
            repo_info = session.repoInfo(opts['repo_id'])
            if not repo_info:
                raise koji.BuildError, 'No such repo: %s' % opts['repo_id']
            repo_state = koji.REPO_STATES[repo_info['state']]
            if repo_state not in ('READY', 'EXPIRED'):
                raise koji.BuildError, 'Bad repo: %s (%s)' % (repo_info['id'], repo_state)
            self.event_id = repo_info['create_event']
        else:
            repo_info = None
            #we'll wait for a repo later (self.getRepo)
            self.event_id = None
        task_info = session.getTaskInfo(self.id)
        target_info = None
        if target:
            target_info = session.getBuildTarget(target, event=self.event_id)
        if target_info:
            dest_tag = target_info['dest_tag']
            build_tag = target_info['build_tag']
            if repo_info is not None:
                #make sure specified repo matches target
                if repo_info['tag_id'] != target_info['build_tag']:
                    raise koji.BuildError, 'Repo/Target mismatch: %s/%s' \
                            % (repo_info['tag_name'], target_info['build_tag_name'])
        else:
            # if repo_id is specified, we can allow the 'target' arg to simply specify
            # the destination tag (since the repo specifies the build tag).
            if repo_info is None:
                raise koji.GenericError, 'unknown build target: %s' % target
            build_tag = repo_info['tag_id']
            if target is None:
                #ok, call it skip-tag for the buildroot tag
                self.opts['skip_tag'] = True
                dest_tag = build_tag
            else:
                taginfo = session.getTag(target, event=self.event_id)
                if not taginfo:
                    raise koji.GenericError, 'neither tag nor target: %s' % target
                dest_tag = taginfo['id']
        #policy checks...
        policy_data = {
            'user_id' : task_info['owner'],
            'source' : src,
            'task_id' : self.id,
            'build_tag' : build_tag,  #id
            'skip_tag' : bool(self.opts.get('skip_tag')),
        }
        if target_info:
            policy_data['target'] = target_info['id'],
        if not self.opts.get('skip_tag'):
            policy_data['tag'] = dest_tag  #id
        if not SCM.is_scm_url(src) and not opts.get('scratch'):
            #let hub policy decide
            session.host.assertPolicy('build_from_srpm', policy_data)
        if opts.get('repo_id') is not None:
            # use of this option is governed by policy
            session.host.assertPolicy('build_from_repo_id', policy_data)
        if not repo_info:
            repo_info = self.getRepo(build_tag)  #(subtask)
            self.event_id = session.getLastEvent()['id']
        srpm = self.getSRPM(src, build_tag, repo_info['id'])
        h = self.readSRPMHeader(srpm)
        data = koji.get_header_fields(h,['name','version','release','epoch'])
        data['task_id'] = self.id
        extra_arches = None
        self.logger.info("Reading package config for %(name)s" % data)
        pkg_cfg = session.getPackageConfig(dest_tag,data['name'],event=self.event_id)
        self.logger.debug("%r" % pkg_cfg)
        if pkg_cfg is not None:
            extra_arches = pkg_cfg.get('extra_arches')
        if not self.opts.get('skip_tag') and not self.opts.get('scratch'):
            # Make sure package is on the list for this tag
            if pkg_cfg is None:
                raise koji.BuildError, "package %s not in list for tag %s" \
                        % (data['name'], target_info['dest_tag_name'])
            elif pkg_cfg['blocked']:
                raise koji.BuildError, "package %s is blocked for tag %s" \
                        % (data['name'], target_info['dest_tag_name'])
            # TODO - more pre tests
        archlist = self.getArchList(build_tag, h, extra=extra_arches)
        #let the system know about the build we're attempting
        if not self.opts.get('scratch'):
            #scratch builds do not get imported
            build_id = session.host.initBuild(data)
        #(initBuild raises an exception if there is a conflict)
        try:
            srpm,rpms,brmap,logs = self.runBuilds(srpm,build_tag,archlist,repo_info['id'])
            if opts.get('scratch'):
                #scratch builds do not get imported
                session.host.moveBuildToScratch(self.id,srpm,rpms,logs=logs)
            else:
                session.host.completeBuild(self.id,build_id,srpm,rpms,brmap,logs=logs)
        except (SystemExit,ServerExit,KeyboardInterrupt):
            #we do not trap these
            raise
        except:
            if not self.opts.get('scratch'):
                #scratch builds do not get imported
                session.host.failBuild(self.id, build_id)
            # reraise the exception
            raise
        if not self.opts.get('skip_tag') and not self.opts.get('scratch'):
            self.tagBuild(build_id,dest_tag)

    def getSRPM(self, src, build_tag, repo_id):
        """Get srpm from src"""
        if isinstance(src,str):
            if SCM.is_scm_url(src):
                return self.getSRPMFromSCM(src, build_tag, repo_id)
            else:
                #assume this is a path under uploads
                return src
        else:
            raise koji.BuildError, 'Invalid source specification: %s' % src
            #XXX - other methods?

    def getSRPMFromSCM(self, url, build_tag, repo_id):
        #TODO - allow different ways to get the srpm
        task_id = session.host.subtask(method='buildSRPMFromSCM',
                                       arglist=[url, build_tag, {'repo_id': repo_id}],
                                       label='srpm',
                                       parent=self.id)
        # wait for subtask to finish
        result = self.wait(task_id)[task_id]
        srpm = result['srpm']
        return srpm

    def readSRPMHeader(self, srpm):
        #srpm arg should be a path relative to <BASEDIR>/work
        global options
        self.logger.debug("Reading SRPM")
        relpath = "work/%s" % srpm
        opts = dict([(k, getattr(options, k)) for k in 'topurl','topdir'])
        fo = koji.openRemoteFile(relpath, **opts)
        h = koji.get_rpm_header(fo)
        if h[rpm.RPMTAG_SOURCEPACKAGE] != 1:
            raise koji.BuildError, "%s is not a source package" % srpm
        return h

    def getArchList(self, build_tag, h, extra=None):
        # get list of arches to build for
        buildconfig = session.getBuildConfig(build_tag, event=self.event_id)
        arches = buildconfig['arches']
        if not arches:
            #XXX - need to handle this better
            raise koji.BuildError, "No arches for tag %(name)s [%(id)s]" % buildconfig
        tag_archlist = [koji.canonArch(a) for a in arches.split()]
        self.logger.debug('arches: %s' % arches)
        if extra:
            self.logger.debug('Got extra arches: %s' % extra)
            arches = "%s %s" % (arches,extra)
        archlist = arches.split()
        self.logger.debug('base archlist: %r' % archlist)
        # - adjust arch list based on srpm macros
        buildarchs = h[rpm.RPMTAG_BUILDARCHS]
        exclusivearch = h[rpm.RPMTAG_EXCLUSIVEARCH]
        excludearch = h[rpm.RPMTAG_EXCLUDEARCH]
        if buildarchs:
            archlist = buildarchs
            self.logger.debug('archlist after buildarchs: %r' % archlist)
        if exclusivearch:
            archlist = [ a for a in archlist if a in exclusivearch ]
            self.logger.debug('archlist after exclusivearch: %r' % archlist)
        if excludearch:
            archlist = [ a for a in archlist if a not in excludearch ]
            self.logger.debug('archlist after excludearch: %r' % archlist)
        #noarch is funny
        if 'noarch' not in excludearch and \
                ( 'noarch' in buildarchs or 'noarch' in exclusivearch ):
            archlist.append('noarch')
        override = self.opts.get('arch_override')
        if self.opts.get('scratch') and override:
            #only honor override for scratch builds
            self.logger.debug('arch override: %s' % override)
            archlist = override.split()
        archdict = {}
        for a in archlist:
            # Filter based on canonical arches for tag
            # This prevents building for an arch that we can't handle
            if a == 'noarch' or koji.canonArch(a) in tag_archlist:
                archdict[a] = 1
        if not archdict:
            raise koji.BuildError, "No matching arches were found"
        return archdict.keys()

    def getRepo(self, tag):
        """Get repo to use for builds"""
        repo_info = session.getRepo(tag)
        if not repo_info:
            #wait for it
            task_id = session.host.subtask(method='waitrepo',
                                           arglist=[tag, None, None],
                                           parent=self.id)
            repo_info = self.wait(task_id)[task_id]
        return repo_info

    def runBuilds(self, srpm, build_tag, archlist, repo_id):
        self.logger.debug("Spawning jobs for arches: %r" % (archlist))
        subtasks = {}
        keep_srpm = True
        for arch in archlist:
            subtasks[arch] = session.host.subtask(method='buildArch',
                                                  arglist=[srpm, build_tag, arch, keep_srpm, {'repo_id': repo_id}],
                                                  label=arch,
                                                  parent=self.id,
                                                  arch=koji.canonArch(arch))
            keep_srpm = False

        self.logger.debug("Got subtasks: %r" % (subtasks))
        self.logger.debug("Waiting on subtasks...")

        # wait for subtasks to finish
        results = self.wait(subtasks.values(), all=True, failany=True)

        # finalize import
        # merge data into needed args for completeBuild call
        rpms = []
        brmap = {}
        logs = {}
        built_srpm = None
        for (arch, task_id) in subtasks.iteritems():
            result = results[task_id]
            self.logger.debug("DEBUG: %r : %r " % (arch,result,))
            brootid = result['brootid']
            for fn in result['rpms']:
                rpms.append(fn)
                brmap[fn] = brootid
            for fn in result['logs']:
                logs.setdefault(arch,[]).append(fn)
            if result['srpms']:
                if built_srpm:
                    raise koji.BuildError, "multiple builds returned a srpm.  task %i" % self.id
                else:
                    built_srpm = result['srpms'][0]
                    brmap[result['srpms'][0]] = brootid
        if built_srpm:
            srpm = built_srpm
        else:
            raise koji.BuildError("could not find a built srpm")

        return srpm,rpms,brmap,logs

    def tagBuild(self,build_id,dest_tag):
        #XXX - need options to skip tagging and to force tagging
        #create the tagBuild subtask
        #this will handle the "post tests"
        task_id = session.host.subtask(method='tagBuild',
                                       arglist=[dest_tag,build_id,False,None,True],
                                       label='tag',
                                       parent=self.id,
                                       arch='noarch')
        self.wait(task_id)

class BuildArchTask(BaseTaskHandler):

    Methods = ['buildArch']

    def weight(self):
        return 1.5

    def updateWeight(self, name):
        """
        Update the weight of this task based on the package we're building.
        weight is scaled from a minimum of 1.5 to a maximum of 6, based on
        the average duration of a build of this package.
        """
        avg = session.getAverageBuildDuration(name)
        if not avg:
            return
        # increase the task weight by 0.75 for every hour of build duration
        adj = (avg / 4800.0)
        # cap the adjustment at +4.5
        weight = self.weight() + min(4.5, adj)
        session.host.setTaskWeight(self.id, weight)

    def srpm_sanity_checks(self, filename):
        header = koji.get_rpm_header(filename)

        if not header[rpm.RPMTAG_PACKAGER]:
            raise koji.BuildError, "The build system failed to set the packager tag"
        if not header[rpm.RPMTAG_VENDOR]:
            raise koji.BuildError, "The build system failed to set the vendor tag"
        if not header[rpm.RPMTAG_DISTRIBUTION]:
            raise koji.BuildError, "The build system failed to set the distribution tag"

    def handler(self, pkg, root, arch, keep_srpm, opts=None):
        """Build a package in a buildroot for one arch"""
        global options
        
        ret = {}
        if opts is None:
            opts = {}
        repo_id = opts.get('repo_id')
        if not repo_id:
            raise koji.BuildError, "A repo id must be provided"
        repo_info = session.repoInfo(repo_id, strict=True)
        event_id = repo_info['create_event']

        # starting srpm should already have been uploaded by parent
        self.logger.debug("Reading SRPM")
        fn = self.localPath("work/%s" % pkg)
        if not os.path.exists(fn):
            raise koji.BuildError, "SRPM file missing: %s" % fn
        # peel E:N-V-R from package
        h = koji.get_rpm_header(fn)
        name = h[rpm.RPMTAG_NAME]
        ver = h[rpm.RPMTAG_VERSION]
        rel = h[rpm.RPMTAG_RELEASE]
        epoch = h[rpm.RPMTAG_EPOCH]
        if h[rpm.RPMTAG_SOURCEPACKAGE] != 1:
            raise koji.BuildError, "not a source package"
        # Disable checking for distribution in the initial SRPM because it
        # might have been built outside of the build system
        # if not h[rpm.RPMTAG_DISTRIBUTION]:
        #    raise koji.BuildError, "the distribution tag is not set in the original srpm"

        self.updateWeight(name)

        rootopts = {
            'repo_id': repo_id
            }
        br_arch = self.find_arch(arch, session.host.getHost(), session.getBuildConfig(root, event=event_id))
        broot = BuildRoot(root, br_arch, self.id, **rootopts)

        self.logger.debug("Initializing buildroot")
        broot.init()

        # run build
        self.logger.debug("Running build")
        broot.build(fn,arch)

        # extract results
        resultdir = broot.resultdir()
        rpm_files = []
        srpm_files = []
        log_files = []
        unexpected = []
        for f in os.listdir(resultdir):
            # files here should have one of two extensions: .log and .rpm
            if f[-4:] == ".log":
                log_files.append(f)
            elif f[-8:] == ".src.rpm":
                srpm_files.append(f)
            elif f[-4:] == ".rpm":
                rpm_files.append(f)
            else:
                unexpected.append(f)
        self.logger.debug("rpms: %r" % rpm_files)
        self.logger.debug("srpms: %r" % srpm_files)
        self.logger.debug("logs: %r" % log_files)
        self.logger.debug("unexpected: %r" % unexpected)

        # upload files to storage server
        uploadpath = broot.getUploadPath()
        for f in rpm_files:
            self.uploadFile("%s/%s" % (resultdir,f))
        self.logger.debug("keep srpm %i %s %s" % (self.id, keep_srpm, opts))
        if keep_srpm:
            if len(srpm_files) == 0:
                raise koji.BuildError, "no srpm files found for task %i" % self.id
            if len(srpm_files) > 1:
                raise koji.BuildError, "mulitple srpm files found for task %i: %s" % (self.id, srpm_files)

            # Run sanity checks.  Any failures will throw a BuildError
            self.srpm_sanity_checks("%s/%s" % (resultdir,srpm_files[0]))

            self.logger.debug("uploading %s/%s to %s" % (resultdir,srpm_files[0], uploadpath))
            self.uploadFile("%s/%s" % (resultdir,srpm_files[0]))
        if rpm_files:
            ret['rpms'] = [ "%s/%s" % (uploadpath,f) for f in rpm_files ]
        else:
            ret['rpms'] = []
        if keep_srpm:
            ret['srpms'] = [ "%s/%s" % (uploadpath,f) for f in srpm_files ]
        else:
            ret['srpms'] = []
        ret['logs'] = [ "%s/%s" % (uploadpath,f) for f in log_files ]

        ret['brootid'] = broot.id

        broot.expire()
        #Let TaskManager clean up

        return ret


class TagBuildTask(BaseTaskHandler):

    Methods = ['tagBuild']
    #XXX - set weight?

    def handler(self, tag_id, build_id, force=False, fromtag=None, ignore_success=False):
        task = session.getTaskInfo(self.id)
        user_id = task['owner']
        try:
            build = session.getBuild(build_id, strict=True)
            tag = session.getTag(tag_id, strict=True)

            #several basic sanity checks have already been run (and will be run
            #again when we make the final call). Our job is to perform the more
            #computationally expensive 'post' tests.

            #XXX - add more post tests
            session.host.tagBuild(self.id,tag_id,build_id,force=force,fromtag=fromtag)
            session.host.tagNotification(True, tag_id, fromtag, build_id, user_id, ignore_success)
        except Exception, e:
            exctype, value = sys.exc_info()[:2]
            session.host.tagNotification(False, tag_id, fromtag, build_id, user_id, ignore_success, "%s: %s" % (exctype, value))
            raise e

# LiveCDTask begins with a mock chroot, and then installs livecd-tools into it
# via the livecd-build group. livecd-creator is then executed in the chroot
# to create the LiveCD image.
#
class LiveCDTask(BaseTaskHandler):

    Methods = ['createLiveCD']
    _taskWeight = 1.5

    def handler(self, arch, target, ksfile, opts=None):
        if not opts:
            opts = {}

        global options
        target_info = session.getBuildTarget(target, strict=True)
        build_tag = target_info['build_tag']
        repo_info = session.getRepo(build_tag)

        # Here we configure mock to bind mount a set of /dev directories
        # so livecd-creator can use them.
        # We use mknod to create the loopN device files later, but if we
        # wanted to have mock do that work, our bind_opts variable would have
        # this additional dictionary:
        #	{ 'files' : {
        #		'/dev/loop0' : '/dev/loop0',
        #		'/dev/loop1' : '/dev/loop1',
        #		'/dev/loop2' : '/dev/loop2',
        #		'/dev/loop3' : '/dev/loop3',
        #		'/dev/loop4' : '/dev/loop4',
        #		'/dev/loop5' : '/dev/loop5',
        #		'/dev/loop6' : '/dev/loop6',
        #		'/dev/loop7' : '/dev/loop7'}}
        bind_opts = {'dirs' : {
                         '/dev/pts' : '/dev/pts',
                         '/dev/mapper' : '/dev/mapper',
                         '/selinux' : '/selinux'}
                     }
        rootopts = {'install_group': 'livecd-build',
                    'setup_dns': True,
                    'repo_id': repo_info['id'],
                    'bind_opts' : bind_opts}

        broot = BuildRoot(build_tag, arch, self.id, **rootopts)

        # create the mock chroot
        self.logger.debug("Initializing buildroot")
        broot.init()

        # Note that if the KS file existed locally, then "ksfile" is a relative
        # path to it in the /mnt/koji/work directory. If not, then it is still
        # the parameter the user passed in initially, and we assume it is a
        # relative path in a remote scm. The user should have passed in an scm
        # url with --ksurl.
        tmpchroot = '/tmp'
        scmdir = os.path.join(broot.rootdir(), tmpchroot[1:])
        koji.ensuredir(scmdir)
        self.logger.debug("ksfile = %s" % ksfile)
        if opts.get('ksurl'):
            scm = SCM(opts['ksurl'])
            scm.assert_allowed(options.allowed_scms)
            logfile = os.path.join(self.workdir, 'checkout.log')
            scmsrcdir = scm.checkout(scmdir, self.getUploadDir(), logfile)
            kspath = os.path.join(scmsrcdir, ksfile)
        else:
            kspath = self.localPath("work/%s" % ksfile)

        # XXX: If the ks file came from a local path and has %include
        #      macros, livecd-creator will fail because the included
        #      kickstarts were not copied into the chroot. For now we
        #      require users to flatten their kickstart file if submitting
        #      the task with a local path.
        #
        #      Note that if an SCM URL was used instead, %include macros
        #      may not be a problem if the included kickstarts are present
        #      in the repository we checked out.

        # Now we do some kickstart manipulation. If the user passed in a repo
        # url with --repo, then we substitute that in for the repo(s) specified
        # in the kickstart file. If --repo wasn't specified, then we use the
        # repo associated with the target passed in initially.
        self.uploadFile(kspath) # upload the original ks file
        if opts.get('ksversion'):
            version = ksparser.makeVersion(ksparser.stringToVersion(opts['ksversion']))
        else:
            version = ksparser.makeVersion()
        ks = ksparser.KickstartParser(version)
        repo_class = kscontrol.dataMap[ks.version]['RepoData']

        try:
            ks.readKickstart(kspath)
        except IOError, (err, msg):
            raise koji.LiveCDError("Failed to read kickstart file "
                                   "'%s' : %s" % (ksfile, msg))
        except kserrors.KickstartError, e:
            raise koji.LiveCDError("Failed to parse kickstart file "
                                    "'%s' : %s" % (ksfile, e))

        ks.handler.repo.repoList = [] # delete whatever the ks file told us
        if opts.get('repo'):
            user_repos = opts['repo'].split(',')
            index = 0
            for user_repo in user_repos:
                ks.handler.repo.repoList.append(repo_class(baseurl=user_repo,
                                                           name='koji-override-%i' % index))
                index += 1
        else:
            topurl = getattr(options, 'topurl')
            if not topurl:
                raise koji.LiveCDError, 'topurl must be defined in kojid.conf'
            path_info = koji.PathInfo(topdir=topurl)
            repopath = path_info.repo(repo_info['id'],
                                      target_info['build_tag_name'])
            baseurl = '%s/%s' % (repopath, arch)
            self.logger.debug('BASEURL: %s' % baseurl)
            ks.handler.repo.repoList.append(repo_class(baseurl=baseurl,
                                                       name='koji-%s-%i' % (target_info['build_tag_name'],
                                                                            repo_info['id'])))

        # Write out the new ks file. Note that things may not be in the same
        # order and comments in the original ks file may be lost.
        kskoji = os.path.join(tmpchroot, 'koji-livecd-%s-%i.ks' % (target_info['build_tag_name'], self.id))
        kspath = os.path.join(broot.rootdir(), kskoji[1:])
        outfile = open(kspath, 'w')
        outfile.write(str(ks.handler))
        outfile.close()

        # put the new ksfile in the output directory
        if not os.path.exists(kspath):
            raise koji.LiveCDError, "KS file missing: %s" % kspath
        self.uploadFile(kspath)

        cachedir = os.path.join(tmpchroot, 'koji-livecd')   # arbitrary
        livecd_log = os.path.join(tmpchroot, 'livecd.log')  # path in chroot

        # Create the loopback devices we need
        mknod_cmd = 'for i in $(seq 0 7); do mknod /dev/loop$i b 7 $i; done'
        rv = broot.mock(['--chroot', mknod_cmd])
        if rv:
            broot.expire()
            raise koji.LiveCDError, \
                "Could not create loopback device files: %s" % _parseStatus(rv, '"%s"' % mknod_cmd)

        # Run livecd-creator
        livecd_cmd = ['/usr/bin/livecd-creator', '-c', kskoji, '-d', '-v', '--logfile', livecd_log,
                      '--cache', cachedir]

        # run the livecd-creator command
        rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + livecd_cmd)
        self.uploadFile(os.path.join(broot.rootdir(), livecd_log[1:]))
        broot.expire()
        if rv:
            raise koji.LiveCDError, \
                "Could not create LiveCD: %s" % _parseStatus(rv, 'livecd-creator') + \
                "; see root.log or livecd.log for more information"

        # Find the resultant iso
        # The cwd of the livecd-creator process is /tmp in the chroot, so that's
        # where it writes the .iso
        files = os.listdir(os.path.join(broot.rootdir(), 'tmp'))
        isofile = None
        for afile in files:
            if afile.endswith('.iso'):
                if not isofile:
                    isofile = afile
                else:
                    raise koji.LiveCDError, 'multiple .iso files found: %s and %s' % (isofile, afile)
        if not isofile:
            raise koji.LiveCDError, 'could not find iso file in chroot'

        isosrc = os.path.join(broot.rootdir(), 'tmp', isofile)
        try:
            filesize = os.path.getsize(isosrc)
        except OSError:
            self.logger.error('Could not find the ISO. Did livecd-creator ' +
                              'complete without errors?')
            raise koji.LiveCDError, 'missing image file: %s' % isosrc

        # if filesize is greater than a 32-bit signed integer's range, the
        # python XMLRPC module will break.
        filesize = str(filesize)

        # copy the iso out of the chroot. If we were given an isoname, this is
        # where the renaming happens.
        self.logger.debug('uploading image: %s' % isosrc)
        if opts.get('isoname'):
            isofile = opts['isoname']
            if not isofile.endswith('.iso'):
                isofile += '.iso'
        self.uploadFile(isosrc, remoteName=isofile)

        # Generate the file manifest of the image
        manifest = os.path.join(broot.resultdir(), 'manifest.log')
        self.genImageManifest(isosrc, manifest)
        self.uploadFile(manifest)

        if not opts.get('scratch'):
            # Read the rpm header information from the yum cache livecd-creator
            # used. We assume it was empty to start.
            #
            # LiveCD creator would have thrown an error if no repo was
            # specified, so we assume the ones we parse are ok.
            repos = os.listdir(os.path.join(broot.rootdir(), cachedir[1:]))
            if len(repos) == 0:
                raise koji.LiveCDError, 'No repos found in yum cache!'

            hdrlist = []
            fields = ['name', 'version', 'release', 'epoch', 'arch', \
                      'buildtime', 'sigmd5']
            for repo in repos:
                pkgpath = os.path.join(broot.rootdir(), cachedir[1:], repo,
                                       'packages')
                pkgs = os.listdir(pkgpath)
                for pkg in pkgs:
                    pkgfile = os.path.join(pkgpath, pkg)
                    hdr = koji.get_header_fields(pkgfile, fields)
                    hdr['size'] = os.path.getsize(pkgfile)
                    hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
                    del hdr['sigmd5']
                    hdrlist.append(hdr)

            # get a unique hash of the image file
            sha256sum = hashlib.sha256()
            image_fo = file(isosrc, 'r')
            while True:
                data = image_fo.read(1048576)
                sha256sum.update(data)
                if not len(data):
                    break

            hash = sha256sum.hexdigest()
            self.logger.debug('digest: %s' % hash)

            # Import info about the image into the database, unless this is a
            # scratch image.
            broot.markExternalRPMs(hdrlist)
            image_id = session.host.importImage(self.id, isofile, filesize, arch,
                                                'LiveCD ISO', hash, hdrlist)

        if opts.get('scratch'):
            return 'Scratch image created: %s' % \
                   os.path.join(koji.pathinfo.work(),
                                koji.pathinfo.taskrelpath(self.id), isofile)
        else:
            return 'Created image: %s' % \
                   os.path.join(koji.pathinfo.imageFinalPath(),
                                koji.pathinfo.livecdRelPath(image_id), isofile)

class BuildSRPMFromSCMTask(BaseTaskHandler):

    Methods = ['buildSRPMFromSCM']
    _taskWeight = 1.0

    def spec_sanity_checks(self, filename):
        spec = open(filename).read()
        for tag in ("Packager", "Distribution", "Vendor"):
            if re.match("%s:" % tag, spec, re.M):
                raise koji.BuildError, "%s is not allowed to be set in spec file" % tag
        for tag in ("packager", "distribution", "vendor"):
            if re.match("%%define\s+%s\s+" % tag, spec, re.M):
                raise koji.BuildError, "%s is not allowed to be defined in spec file" % tag

    def handler(self, url, build_tag, opts=None):
        global options

        # will throw a BuildError if the url is invalid
        scm = SCM(url)
        scm.assert_allowed(options.allowed_scms)

        if opts is None:
            opts = {}
        repo_id = opts.get('repo_id')
        if not repo_id:
            raise koji.BuildError, "A repo id must be provided"

        repo_info = session.repoInfo(repo_id, strict=True)
        event_id = repo_info['create_event']
        build_tag = session.getTag(build_tag, strict=True, event=event_id)

        # need DNS in the chroot because "make srpm" may need to contact
        # a SCM or lookaside cache to retrieve the srpm contents
        rootopts = {'install_group': 'srpm-build',
                    'setup_dns': True,
                    'repo_id': repo_id}
        br_arch = self.find_arch('noarch', session.host.getHost(), session.getBuildConfig(build_tag['id'], event=event_id))
        broot = BuildRoot(build_tag['id'], br_arch, self.id, **rootopts)

        self.logger.debug("Initializing buildroot")
        broot.init()

        # Setup files and directories for SRPM creation
        # We can't put this under the mock homedir because that directory
        # is completely blown away and recreated on every mock invocation
        scmdir = broot.rootdir() + '/tmp/scmroot'
        koji.ensuredir(scmdir)
        logfile = self.workdir + '/checkout.log'
        uploadpath = self.getUploadDir()

        # Check out spec file, etc. from SCM
        sourcedir = scm.checkout(scmdir, uploadpath, logfile)
        # chown the sourcedir and everything under it to the mockuser
        # so we can build the srpm as non-root
        uid = pwd.getpwnam(options.mockuser)[2]
        # rpmbuild seems to complain if it's running in the "mock" group but
        # files are in a different group
        gid = grp.getgrnam('mock')[2]
        for dirpath, dirnames, filenames in os.walk(scmdir):
            os.chown(dirpath, uid, gid)
            for filename in filenames:
                os.chown(os.path.join(dirpath, filename), uid, gid)

        # Find and verify that there is only one spec file.
        spec_files = glob.glob("%s/*.spec" % sourcedir)
        if len(spec_files) == 0:
            raise koji.BuildError("No spec file found")
        elif len(spec_files) > 1:
            raise koji.BuildError("Multiple spec files found: %s" % spec_files)
        spec_file = spec_files[0]

        # Run spec file sanity checks.  Any failures will throw a BuildError
        self.spec_sanity_checks(spec_file)

        #build srpm
        self.logger.debug("Running srpm build")
        broot.build_srpm(spec_file, sourcedir)

        srpms = glob.glob('%s/*.src.rpm' % broot.resultdir())
        if len(srpms) == 0:
            raise koji.BuildError, "No srpms found in %s" % sourcedir
        elif len(srpms) > 1:
            raise koji.BuildError, "Multiple srpms found in %s: %s" % (sourcedir, ", ".join(srpms))
        else:
            srpm = srpms[0]
        
        # check srpm name
        h = koji.get_rpm_header(srpm)
        name = h[rpm.RPMTAG_NAME]
        version = h[rpm.RPMTAG_VERSION]
        release = h[rpm.RPMTAG_RELEASE]
        srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals()
        if srpm_name != os.path.basename(srpm):
            raise koji.BuildError, 'srpm name mismatch: %s != %s' % (srpm_name, os.path.basename(srpm))

        #upload srpm and return
        self.uploadFile(srpm)

        broot.expire()

        return {'srpm': "%s/%s" % (uploadpath, srpm_name)}

class TagNotificationTask(BaseTaskHandler):
    Methods = ['tagNotification']

    _taskWeight = 0.1

    message_templ = \
"""From: %(from_addr)s\r
Subject: %(nvr)s %(result)s %(operation)s by %(user_name)s\r
To: %(to_addrs)s\r
X-Koji-Package: %(pkg_name)s\r
X-Koji-NVR: %(nvr)s\r
X-Koji-User: %(user_name)s\r
X-Koji-Status: %(status)s\r
%(tag_headers)s\r
\r
Package: %(pkg_name)s\r
NVR: %(nvr)s\r
User: %(user_name)s\r
Status: %(status)s\r
%(operation_details)s\r
%(nvr)s %(result)s %(operation)s by %(user_name)s\r
%(failure_info)s\r
"""

    def handler(self, recipients, is_successful, tag_info, from_info, build_info, user_info, ignore_success=None, failure_msg=''):
        if len(recipients) == 0:
            self.logger.debug('task %i: no recipients, not sending notifications', self.id)
            return

        if ignore_success and is_successful:
            self.logger.debug('task %i: tag operation successful and ignore success is true, not sending notifications', self.id)
            return

        build = session.getBuild(build_info)
        user = session.getUser(user_info)
        pkg_name = build['package_name']
        nvr = koji.buildLabel(build)
        user_name = user['name']

        from_addr = options.from_addr
        to_addrs = ', '.join(recipients)

        operation = '%(action)s'
        operation_details = 'Tag Operation: %(action)s\r\n'
        tag_headers = ''
        if from_info:
            from_tag = session.getTag(from_info)
            from_tag_name = from_tag['name']
            operation += ' from %s' % from_tag_name
            operation_details += 'From Tag: %s\r\n' % from_tag_name
            tag_headers += 'X-Koji-Tag: %s' % from_tag_name
            action = 'untagged'
        if tag_info:
            tag = session.getTag(tag_info)
            tag_name = tag['name']
            operation += ' into %s' % tag_name
            operation_details += 'Into Tag: %s\r\n' % tag_name
            if tag_headers:
                tag_headers += '\r\n'
            tag_headers += 'X-Koji-Tag: %s' % tag_name
            action = 'tagged'
        if tag_info and from_info:
            action = 'moved'
        operation = operation % locals()
        operation_details = operation_details % locals()

        if is_successful:
            result = 'successfully'
            status = 'complete'
            failure_info = ''
        else:
            result = 'unsuccessfully'
            status = 'failed'
            failure_info = "Operation failed with the error:\r\n    %s\r\n" % failure_msg

        message = self.message_templ % locals()
        # ensure message is in UTF-8
        message = message.encode('utf-8')

        server = smtplib.SMTP(options.smtphost)
        #server.set_debuglevel(True)
        server.sendmail(from_addr, recipients, message)
        server.quit()

        return 'sent notification of tag operation %i to: %s' % (self.id, to_addrs)

class BuildNotificationTask(BaseTaskHandler):
    Methods = ['buildNotification']

    _taskWeight = 0.1

    # XXX externalize these templates somewhere
    subject_templ = """Package: %(build_nvr)s Tag: %(dest_tag)s Status: %(status)s Built by: %(build_owner)s"""
    message_templ = \
"""From: %(from_addr)s\r
Subject: %(subject)s\r
To: %(to_addrs)s\r
X-Koji-Tag: %(dest_tag)s\r
X-Koji-Package: %(build_pkg_name)s\r
X-Koji-Builder: %(build_owner)s\r
X-Koji-Status: %(status)s\r
\r
Package: %(build_nvr)s\r
Tag: %(dest_tag)s\r
Status: %(status)s%(cancel_info)s\r
Built by: %(build_owner)s\r
ID: %(build_id)i\r
Started: %(creation_time)s\r
Finished: %(completion_time)s\r
%(changelog)s\r
%(failure)s\r
%(output)s\r
Task Info: %(weburl)s/taskinfo?taskID=%(task_id)i\r
Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r
"""

    def _getTaskData(self, task_id, data={}):
        taskinfo = session.getTaskInfo(task_id)
        
        if not taskinfo:
            # invalid task_id
            return data
        
        if taskinfo['host_id']:
            hostinfo = session.getHost(taskinfo['host_id'])
        else:
            hostinfo = None
        
        result = None
        try:
            result = session.getTaskResult(task_id)
        except:
            excClass, result = sys.exc_info()[:2]
            if hasattr(result, 'faultString'):
                result = result.faultString
            else:
                result = '%s: %s' % (excClass.__name__, result)
            result = result.strip()
            # clear the exception, since we're just using
            # it for display purposes
            sys.exc_clear()
        if not result:
            result = 'Unknown'
        
        files = session.listTaskOutput(task_id)
        logs = [filename for filename in files if filename.endswith('.log')]
        rpms = [filename for filename in files if filename.endswith('.rpm') and not filename.endswith('.src.rpm')]
        srpms = [filename for filename in files if filename.endswith('.src.rpm')]
        misc = [filename for filename in files if filename not in logs + rpms + srpms]
        
        logs.sort()
        rpms.sort()
        misc.sort()
        
        data[task_id] = {}
        data[task_id]['id'] = taskinfo['id']
        data[task_id]['method'] = taskinfo['method']
        data[task_id]['arch'] = taskinfo['arch']
        data[task_id]['build_arch'] = taskinfo['label']
        data[task_id]['host'] = hostinfo and hostinfo['name'] or None
        data[task_id]['state'] = koji.TASK_STATES[taskinfo['state']].lower()
        data[task_id]['result'] = result
        data[task_id]['request'] = session.getTaskRequest(task_id)
        data[task_id]['logs'] = logs
        data[task_id]['rpms'] = rpms
        data[task_id]['srpms'] = srpms
        data[task_id]['misc'] = misc
        
        children = session.getTaskChildren(task_id)
        for child in children:
            data = self._getTaskData(child['id'], data)
        return data

    def handler(self, recipients, build, target, weburl):
        if len(recipients) == 0:
            self.logger.debug('task %i: no recipients, not sending notifications', self.id)
            return

        build_pkg_name = build['package_name']
        build_pkg_evr = '%s%s-%s' % ((build['epoch'] and str(build['epoch']) + ':' or ''), build['version'], build['release'])
        build_nvr = koji.buildLabel(build)
        build_id = build['id']
        build_owner = build['owner_name']
        # target comes from session.py:_get_build_target()
        dest_tag = None
        if target is not None:
            dest_tag = target['dest_tag_name']
        status = koji.BUILD_STATES[build['state']].lower()
        creation_time = koji.formatTimeLong(build['creation_time'])
        completion_time = koji.formatTimeLong(build['completion_time'])
        task_id = build['task_id']

        task_data = self._getTaskData(task_id)

        cancel_info = ''
        failure_info = ''
        if build['state'] == koji.BUILD_STATES['CANCELED']:
            # The owner of the buildNotification task is the one
            # who canceled the task, it turns out.
            this_task = session.getTaskInfo(self.id)
            if this_task['owner']:
                canceler = session.getUser(this_task['owner'])
                cancel_info = "\r\nCanceled by: %s" % canceler['name']
        elif build['state'] == koji.BUILD_STATES['FAILED']:
            failure_data = task_data[task_id]['result']
            failed_hosts = ['%s (%s)' % (task['host'], task['arch']) for task in task_data.values() if task['host'] and task['state'] == 'failed']
            failure_info = "\r\n%s (%d) failed on %s:\r\n  %s" % (build_nvr, build_id, 
                                                                  ', '.join(failed_hosts),
                                                                  failure_data)

        failure = failure_info or cancel_info or ''

        tasks = {'failed' : [task for task in task_data.values() if task['state'] == 'failed'],
                 'canceled' : [task for task in task_data.values() if task['state'] == 'canceled'],
                 'closed' : [task for task in task_data.values() if task['state'] == 'closed']}
        
        srpms = []
        for taskinfo in task_data.values():
            for srpmfile in taskinfo['srpms']:
                srpms.append(srpmfile)
        srpms = self.uniq(srpms)
        srpms.sort()
        
        if srpms:
            output = "SRPMS:\r\n"
            for srpm in srpms:
                output += "  %s" % srpm
            output += "\r\n\r\n"
        else:
            output = ''
        
        # list states here to make them go in the correct order
        for task_state in ['failed', 'canceled', 'closed']:
            if tasks[task_state]:
                output += "%s tasks:\r\n" % task_state.capitalize()
                output += "%s-------\r\n\r\n" % ("-" * len(task_state))
                for task in tasks[task_state]:
                    output += "Task %s" % task['id']
                    if task['host']:
                        output += " on %s\r\n" % task['host']
                    else:
                        output += "\r\n"
                    output += "Task Type: %s\r\n" % koji.taskLabel(task)
                    for filetype in ['logs', 'rpms', 'misc']:
                        if task[filetype]:
                            output += "%s:\r\n" % filetype
                            for file in task[filetype]:
                                if filetype == 'rpms':
                                    output += " %s\r\n" % '/'.join([options.pkgurl, build['name'], build['version'], build['release'], task['build_arch'], file])
                                elif filetype == 'logs':
                                    if tasks[task_state] != 'closed':
                                        output += "  %s/getfile?taskID=%s&name=%s\r\n" % (weburl, task['id'], file)
                                    else:
                                        output += " %s\r\n" % '/'.join([options.pkgurl, build['name'], build['version'], build['release'], 'data', 'logs', task['build_arch'], file])
                                elif task[filetype] == 'misc':
                                    output += "  %s/getfile?taskID=%s&name=%s\r\n" % (weburl, task['id'], file)
                    output += "\r\n"
                output += "\r\n"

        changelog = koji.util.formatChangelog(session.getChangelogEntries(build_id, queryOpts={'limit': 3})).replace("\n","\r\n")
        if changelog:
            changelog = "Changelog:\r\n%s" % changelog

        from_addr = options.from_addr
        to_addrs = ', '.join(recipients)
        subject = self.subject_templ % locals()
        message = self.message_templ % locals()
        # ensure message is in UTF-8
        message = message.encode('utf-8')

        server = smtplib.SMTP(options.smtphost)
        # server.set_debuglevel(True)
        server.sendmail(from_addr, recipients, message)
        server.quit()

        return 'sent notification of build %i to: %s' % (build_id, to_addrs)

    def uniq(self, items):
        """Remove duplicates from the list of items, and sort the list."""
        m = dict(zip(items, [1] * len(items)))
        l = m.keys()
        l.sort()
        return l


class NewRepoTask(BaseTaskHandler):
    Methods = ['newRepo']
    _taskWeight = 0.1

    def handler(self, tag, event=None, src=False, debuginfo=False):
        self.uploadpath = self.getUploadDir()
        tinfo = session.getTag(tag, strict=True, event=event)
        kwargs = {}
        if event is not None:
            kwargs['event'] = event
        if src:
            kwargs['with_src'] = True
        if debuginfo:
            kwargs['with_debuginfo'] = True
        repo_id, event_id = session.host.repoInit(tinfo['id'], **kwargs)
        path = koji.pathinfo.repo(repo_id, tinfo['name'])
        if not os.path.isdir(path):
            raise koji.GenericError, "Repo directory missing: %s" % path
        arches = []
        for fn in os.listdir(path):
            if fn != 'groups' and os.path.isfile("%s/%s/pkglist" % (path, fn)):
                arches.append(fn)
        #see if we can find a previous repo to update from
        oldrepo = session.getRepo(tinfo['id'], state=koji.REPO_READY)
        subtasks = {}
        external_repos = session.getExternalRepoList(tinfo['id'], event=event)
        for arch in arches:
            arglist = [repo_id, arch, oldrepo]
            if external_repos:
                arglist.append(external_repos)
            subtasks[arch] = session.host.subtask(method='createrepo',
                                                  arglist=arglist,
                                                  label=arch,
                                                  parent=self.id,
                                                  arch='noarch')
        # wait for subtasks to finish
        results = self.wait(subtasks.values(), all=True, failany=True)
        data = {}
        for (arch, task_id) in subtasks.iteritems():
            data[arch] = results[task_id]
            self.logger.debug("DEBUG: %r : %r " % (arch,data[arch],))
        kwargs = {}
        if event is not None:
            kwargs['expire'] = True
        session.host.repoDone(repo_id, data, **kwargs)
        return repo_id, event_id

class CreaterepoTask(BaseTaskHandler):

    Methods = ['createrepo']
    _taskWeight = 1.5

    def handler(self, repo_id, arch, oldrepo, external_repos=None):
        #arch is the arch of the repo, not the task
        rinfo = session.repoInfo(repo_id, strict=True)
        if rinfo['state'] != koji.REPO_INIT:
            raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo
        self.repo_id = rinfo['id']
        self.pathinfo = koji.PathInfo(options.topdir)
        toprepodir = self.pathinfo.repo(repo_id, rinfo['tag_name'])
        self.repodir = '%s/%s' % (toprepodir, arch)
        if not os.path.isdir(self.repodir):
            raise koji.GenericError, "Repo directory missing: %s" % self.repodir
        groupdata = os.path.join(toprepodir, 'groups', 'comps.xml')
        #set up our output dir
        self.outdir = '%s/repo' % self.workdir
        self.datadir = '%s/repodata' % self.outdir
        pkglist = os.path.join(self.repodir, 'pkglist')
        if os.path.getsize(pkglist) > 0:
            # don't call createrepo with an empty pkglist or it'll
            # add every Koji-managed rpm to the repodata
            self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo)

        external_repos = session.getExternalRepoList(rinfo['tag_id'], event=rinfo['create_event'])
        if external_repos:
            self.merge_repos(external_repos, arch, groupdata)

        uploadpath = self.getUploadDir()
        files = []
        for f in os.listdir(self.datadir):
            files.append(f)
            session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f)

        return [uploadpath, files]

    def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):
        koji.ensuredir(self.outdir)
        cmd = ['/usr/bin/createrepo', '-vd', '-o', self.outdir, '-i', pkglist, '-u', options.pkgurl]
        if os.path.isfile(groupdata):
            cmd.extend(['-g', groupdata])
        #attempt to recycle repodata from last repo
        if oldrepo and options.createrepo_update:
            oldpath = self.pathinfo.repo(oldrepo['id'], rinfo['tag_name'])
            olddatadir = '%s/%s/repodata' % (oldpath, arch)
            if not os.path.isdir(olddatadir):
                self.logger.warn("old repodata is missing: %s" % olddatadir)
            else:
                shutil.copytree(olddatadir, self.datadir)
                oldorigins = os.path.join(self.datadir, 'pkgorigins.gz')
                if os.path.isfile(oldorigins):
                    # remove any previous origins file and rely on mergerepos
                    # to rewrite it (if we have external repos to merge)
                    os.unlink(oldorigins)
                cmd.append('--update')
                if options.createrepo_skip_stat:
                    cmd.append('--skip-stat')
        # note: we can't easily use a cachedir because we do not have write
        # permission. The good news is that with --update we won't need to
        # be scanning many rpms.
        pkgdir = os.path.join(self.pathinfo.topdir, 'packages/')
        cmd.append(pkgdir)

        logfile = '%s/createrepo.log' % self.workdir
        status = log_output(cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
        if not _isSuccess(status):
            raise koji.GenericError, 'failed to create repo: %s' \
                    % _parseStatus(status, ' '.join(cmd))

    def merge_repos(self, external_repos, arch, groupdata):
        repos = []
        if os.path.isdir(self.datadir):
            localdir = '%s/repo_%s_premerge' % (self.workdir, self.repo_id)
            os.rename(self.outdir, localdir)
            koji.ensuredir(self.outdir)
            repos.append('file://' + localdir + '/')

        for repo in external_repos:
            ext_url = repo['url']
            # substitute $arch in the url with the arch of the repo we're generating
            ext_url = ext_url.replace('$arch', arch)
            repos.append(ext_url)

        blocklist = self.repodir + '/blocklist'
        cmd = ['/usr/libexec/kojid/mergerepos', '-a', arch, '-b', blocklist, '-o', self.outdir]
        if os.path.isfile(groupdata):
            cmd.extend(['-g', groupdata])
        for repo in repos:
            cmd.extend(['-r', repo])

        logfile = '%s/mergerepos.log' % self.workdir
        status = log_output(cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
        if not _isSuccess(status):
            raise koji.GenericError, 'failed to merge repos: %s' \
                % _parseStatus(status, ' '.join(cmd))

class WaitrepoTask(BaseTaskHandler):

    Methods = ['waitrepo']
    #mostly just waiting
    _taskWeight = 0.2

    PAUSE = 60
    # time in minutes before we fail this task
    TIMEOUT = 120

    def handler(self, tag, newer_than=None, nvrs=None):
        """Wait for a repo for the tag, subject to given conditions

        newer_than: create_event timestamp should be newer than this
        nvr: repo should contain this nvr (which may not exist at first)

        Only one of the options may be specified. If neither is, then
        the call will wait for the first ready repo.

        Returns the repo info (from getRepo) of the chosen repo
        """

        start = time.time()

        taginfo = session.getTag(tag, strict=True)
        targets = session.getBuildTargets(buildTagID=taginfo['id'])
        if not targets:
            raise koji.GenericError("No build target for tag: %s" % taginfo['name'])

        if isinstance(newer_than, basestring) and newer_than.lower() == "now":
            newer_than = start
        if not isinstance(newer_than, (type(None), int, long, float)):
            raise koji.GenericError, "Invalid value for newer_than: %s" % newer_than

        if newer_than and nvrs:
            raise koji.GenericError, "only one of (newer_than, nvrs) may be specified"

        if not nvrs:
            nvrs = []
        builds = [koji.parse_NVR(nvr) for nvr in nvrs]

        last_repo = None

        while True:
            repo = session.getRepo(taginfo['id'])
            if repo and repo != last_repo:
                if builds:
                    if koji.util.checkForBuilds(session, taginfo['id'], builds, repo['create_event']):
                        self.logger.debug("Successfully waited %s for %s to appear in the %s repo" % \
                            (koji.util.duration(start), koji.util.printList(nvrs), taginfo['name']))
                        return repo
                elif newer_than:
                    if repo['create_ts'] > newer_than:
                        self.logger.debug("Successfully waited %s for a new %s repo" % \
                            (koji.util.duration(start), taginfo['name']))
                        return repo
                else:
                    #no check requested -- return first ready repo
                    return repo

            if (time.time() - start) > (self.TIMEOUT * 60.0):
                if builds:
                    raise koji.GenericError, "Unsuccessfully waited %s for %s to appear in the %s repo" % \
                        (koji.util.duration(start), koji.util.printList(nvrs), taginfo['name'])
                else:
                    raise koji.GenericError, "Unsuccessfully waited %s for a new %s repo" % \
                        (koji.util.duration(start), taginfo['name'])

            time.sleep(self.PAUSE)
            last_repo = repo

class SCM(object):
    "SCM abstraction class"

    types = { 'CVS': ('cvs://',),
              'CVS+SSH': ('cvs+ssh://',),
              'GIT': ('git://', 'git+http://', 'git+https://', 'git+rsync://'),
              'GIT+SSH': ('git+ssh://',),
              'SVN': ('svn://', 'svn+http://', 'svn+https://'),
              'SVN+SSH': ('svn+ssh://',) }

    def is_scm_url(url):
        """
        Return True if the url appears to be a valid, accessible source location, False otherwise
        """
        for schemes in SCM.types.values():
            for scheme in schemes:
                if url.startswith(scheme):
                    return True
        else:
            return False
    is_scm_url = staticmethod(is_scm_url)

    def __init__(self, url):
        """
        Initialize the SCM object using the specified url.
        The expected url format is:

        scheme://[user@]host/path/to/repo?path/to/module#revision_or_tag_identifier

        The initialized SCM object will have the following attributes:
        - url (the unmodified url)
        - scheme
        - user (may be null)
        - host
        - repository
        - module
        - revision
        - use_common (defaults to True, may be set by assert_allowed())
        - scmtype

        The exact format of each attribute is SCM-specific, but the structure of the url
        must conform to the template above, or an error will be raised.
        """
        self.logger = logging.getLogger('koji.build.SCM')

        if not SCM.is_scm_url(url):
            raise koji.GenericError, 'Invalid SCM URL: %s' % url

        self.url = url
        scheme, user, host, path, query, fragment = self._parse_url()

        self.scheme = scheme
        self.user = user
        self.host = host
        self.repository = path
        self.module = query
        self.revision = fragment
        self.use_common = True

        for scmtype, schemes in SCM.types.items():
            if self.scheme in schemes:
                self.scmtype = scmtype
                break
        else:
            # should never happen
            raise koji.GenericError, 'Invalid SCM URL: %s' % url

    def _parse_url(self):
        """
        Parse the SCM url into usable components.
        Return the following tuple:

        (scheme, user, host, path, query, fragment)

        user may be None, everything else will have a value
        """
        # get the url's scheme
        scheme = self.url.split('://')[0] + '://'

        # replace the scheme with http:// so that the urlparse works in all cases
        dummyurl = self.url.replace(scheme, 'http://', 1)
        dummyscheme, netloc, path, params, query, fragment = urlparse.urlparse(dummyurl)

        user = None
        userhost = netloc.split('@')
        if len(userhost) == 2:
            user = userhost[0]
            if not user:
                # Don't return an empty string
                user = None
            elif ':' in user:
                raise koji.GenericError, 'username:password format not supported: %s' % user
            netloc = userhost[1]
        elif len(userhost) > 2:
            raise koji.GenericError, 'Invalid username@hostname specified: %s' % netloc

        # ensure that path and query do not end in /
        if path.endswith('/'):
            path = path[:-1]
        if query.endswith('/'):
            query = query[:-1]

        # check for validity: params should be empty, query may be empty, everything else should be populated
        if params or not (scheme and netloc and path and fragment):
            raise koji.GenericError, 'Unable to parse SCM URL: %s' % self.url

        # return parsed values
        return (scheme, user, netloc, path, query, fragment)

    def assert_allowed(self, allowed):
        """
        Verify that the host and repository of this SCM is in the provided list of
        allowed repositories.

        allowed is a space-separated list of host:repository[:use_common] tuples.  Incorrectly-formatted
        tuples will be ignored.

        If use_common is not present, kojid will attempt to checkout a common/ directory from the
        repository.  If use_common is set to no, off, false, or 0, it will not attempt to checkout a common/
        directory.
        """
        for allowed_scm in allowed.split():
            scm_tuple = allowed_scm.split(':')
            if len(scm_tuple) in (2, 3):
                if fnmatch(self.host, scm_tuple[0]) and fnmatch(self.repository, scm_tuple[1]):
                    # SCM host:repository is in the allowed list
                    # check if we specify a value for use_common
                    if len(scm_tuple) == 3:
                        if scm_tuple[2].lower() in ('no', 'off', 'false', '0'):
                            self.use_common = False
                    break
            else:
                self.logger.warn('Ignoring incorrectly formatted SCM host:repository: %s' % allowed_scm)
        else:
            raise koji.BuildError, '%s:%s is not in the list of allowed SCMs' % (self.host, self.repository)

    def checkout(self, scmdir, uploadpath, logfile):
        """
        Checkout the module from SCM.  Accepts the following parameters:

         - scmdir: the working directory
         - uploadpath: the path on the server the logfile should be uploaded to
         - logfile: the file used for logging command output

        Returns the directory that the module was checked-out into (a subdirectory of scmdir)
        """
        # TODO: sanity check arguments
        sourcedir = '%s/%s' % (scmdir, self.module)

        update_checkout_cmd = None
        update_checkout_dir = None
        env = None

        if self.scmtype == 'CVS':
            pserver = ':pserver:%s@%s:%s' % ((self.user or 'anonymous'), self.host, self.repository)
            module_checkout_cmd = ['cvs', '-d', pserver, 'checkout', '-r', self.revision, self.module]
            common_checkout_cmd = ['cvs', '-d', pserver, 'checkout', 'common']

        elif self.scmtype == 'CVS+SSH':
            if not self.user:
                raise koji.BuildError, 'No user specified for repository access scheme: %s' % self.scheme

            cvsserver = ':ext:%s@%s:%s' % (self.user, self.host, self.repository)
            module_checkout_cmd = ['cvs', '-d', cvsserver, 'checkout', '-r', self.revision, self.module]
            common_checkout_cmd = ['cvs', '-d', cvsserver, 'checkout', 'common']
            env = {'CVS_RSH': 'ssh'}

        elif self.scmtype == 'GIT':
            scheme = self.scheme
            if '+' in scheme:
                scheme = scheme.split('+')[1]
            gitrepo = '%s%s%s' % (scheme, self.host, self.repository)
            commonrepo = os.path.dirname(gitrepo) + '/common'
            checkout_path = os.path.basename(self.repository)
            if self.repository.endswith('/.git'):
                # If we're referring to the .git subdirectory of the main module,
                # assume we need to do the same for the common module
                checkout_path = os.path.basename(self.repository[:-5])
                commonrepo = os.path.dirname(gitrepo[:-5]) + '/common/.git'
            elif self.repository.endswith('.git'):
                # If we're referring to a bare repository for the main module,
                # assume we need to do the same for the common module
                checkout_path = os.path.basename(self.repository[:-4])
                commonrepo = os.path.dirname(gitrepo[:-4]) + '/common.git'

            module_checkout_cmd = ['git', 'clone', '-n', gitrepo, checkout_path]
            common_checkout_cmd = ['git', 'clone', commonrepo, 'common']
            update_checkout_cmd = ['git', 'reset', '--hard', self.revision]
            update_checkout_dir = '%s/%s' % (scmdir, checkout_path)

            sourcedir = '%s/%s' % (scmdir, checkout_path)
            # self.module may be empty, in which case the specfile should be in the top-level directory
            if self.module:
                # Treat the module as a directory inside the git repository
                sourcedir = '%s/%s' % (sourcedir, self.module)

        elif self.scmtype == 'GIT+SSH':
            if not self.user:
                raise koji.BuildError, 'No user specified for repository access scheme: %s' % self.scheme
            gitrepo = 'git+ssh://%s@%s%s' % (self.user, self.host, self.repository)
            commonrepo = os.path.dirname(gitrepo) + '/common'
            checkout_path = os.path.basename(self.repository)
            if self.repository.endswith('/.git'):
                # If we're referring to the .git subdirectory of the main module,
                # assume we need to do the same for the common module
                checkout_path = os.path.basename(self.repository[:-5])
                commonrepo = os.path.dirname(gitrepo[:-5]) + '/common/.git'
            elif self.repository.endswith('.git'):
                # If we're referring to a bare repository for the main module,
                # assume we need to do the same for the common module
                checkout_path = os.path.basename(self.repository[:-4])
                commonrepo = os.path.dirname(gitrepo[:-4]) + '/common.git'

            module_checkout_cmd = ['git', 'clone', '-n', gitrepo, checkout_path]
            common_checkout_cmd = ['git', 'clone', commonrepo, 'common']
            update_checkout_cmd = ['git', 'reset', '--hard', self.revision]
            update_checkout_dir = '%s/%s' % (scmdir, checkout_path)

            sourcedir = '%s/%s' % (scmdir, checkout_path)
            # self.module may be empty, in which case the specfile should be in the top-level directory
            if self.module:
                # Treat the module as a directory inside the git repository
                sourcedir = '%s/%s' % (sourcedir, self.module)

        elif self.scmtype == 'SVN':
            scheme = self.scheme
            if '+' in scheme:
                scheme = scheme.split('+')[1]

            svnserver = '%s%s%s' % (scheme, self.host, self.repository)
            module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module]
            common_checkout_cmd = ['svn', 'checkout', '%s/common' % svnserver]

        elif self.scmtype == 'SVN+SSH':
            if not self.user:
                raise koji.BuildError, 'No user specified for repository access scheme: %s' % self.scheme

            svnserver = 'svn+ssh://%s@%s%s' % (self.user, self.host, self.repository)
            module_checkout_cmd = ['svn', 'checkout', '-r', self.revision, '%s/%s' % (svnserver, self.module), self.module]
            common_checkout_cmd = ['svn', 'checkout', '%s/common' % svnserver]

        else:
            raise koji.BuildError, 'Unknown SCM type: %s' % self.scmtype

        # perform checkouts
        if log_output(module_checkout_cmd[0], module_checkout_cmd, logfile, uploadpath, cwd=scmdir, logerror=1, env=env):
            raise koji.BuildError, 'Error running %s checkout command "%s", see %s for details' % \
                (self.scmtype, ' '.join(module_checkout_cmd), os.path.basename(logfile))

        if update_checkout_cmd:
            # Currently only required for GIT checkouts
            # Run the command in the directory the source was checked out into
            if log_output(update_checkout_cmd[0], update_checkout_cmd, logfile, uploadpath, cwd=update_checkout_dir,
                          logerror=1, append=1, env=env):
                raise koji.BuildError, 'Error running %s update command "%s", see %s for details' % \
                    (self.scmtype, ' '.join(update_checkout_cmd), os.path.basename(logfile))

        if self.use_common:
            if log_output(common_checkout_cmd[0], common_checkout_cmd, logfile, uploadpath, cwd=scmdir, logerror=1, append=1, env=env):
                raise koji.BuildError, 'Error running %s checkout command "%s", see %s for details' % \
                    (self.scmtype, ' '.join(common_checkout_cmd), os.path.basename(logfile))
            if not os.path.exists('%s/../common' % sourcedir):
                # find the relative distance from sourcedir/../common to scmdir/common
                destdir = os.path.split(sourcedir)[0]
                path_comps = destdir[len(scmdir) + 1:]
                rel_path = '../' * len(path_comps.split('/'))
                os.symlink(rel_path + 'common', '%s/../common' % sourcedir)

        return sourcedir

def get_options():
    """process options from command line and config file"""
    global options
    # parse command line args
    parser = OptionParser()
    parser.add_option("-c", "--config", dest="configFile",
                      help="use alternate configuration file", metavar="FILE",
                      default="/etc/kojid/kojid.conf")
    parser.add_option("--user", help="specify user")
    parser.add_option("--password", help="specify password")
    parser.add_option("-f", "--fg", dest="daemon",
                      action="store_false", default=True,
                      help="run in foreground")
    parser.add_option("--force-lock", action="store_true", default=False,
                      help="force lock for exclusive session")
    parser.add_option("-v", "--verbose", action="store_true", default=False,
                      help="show verbose output")
    parser.add_option("-d", "--debug", action="store_true", default=False,
                      help="show debug output")
    parser.add_option("--debug-task", action="store_true", default=False,
                      help="enable debug output for tasks")
    parser.add_option("--debug-xmlrpc", action="store_true", default=False,
                      help="show xmlrpc debug output")
    parser.add_option("--debug-mock", action="store_true", default=False,
                      help="show mock debug output")
    parser.add_option("--skip-main", action="store_true", default=False,
                      help="don't actually run main")
    parser.add_option("--maxjobs", type='int', help="Specify maxjobs")
    parser.add_option("--minspace", type='int', help="Specify minspace")
    parser.add_option("--sleeptime", type='int', help="Specify the polling interval")
    parser.add_option("--admin-emails", help="Address(es) to send error notices to")
    parser.add_option("--topdir", help="Specify topdir")
    parser.add_option("--topurl", help="Specify topurl")
    parser.add_option("--workdir", help="Specify workdir")
    parser.add_option("--pluginpath", help="Specify plugin search path")
    parser.add_option("--plugin", action="append", help="Load specified plugin")
    parser.add_option("--mockdir", help="Specify mockdir")
    parser.add_option("--mockuser", help="User to run mock as")
    parser.add_option("-s", "--server", help="url of XMLRPC server")
    parser.add_option("--pkgurl", help="url of packages directory")
    (options, args) = parser.parse_args()

    if args:
        parser.error("incorrect number of arguments")
        #not reached
        assert False

    # load local config
    config = ConfigParser()
    config.read(options.configFile)
    for x in config.sections():
        if x != 'kojid':
            quit('invalid section found in config file: %s' % x)
    defaults = {'sleeptime': 15,
                'maxjobs': 10,
                'minspace': 8192,
                'admin_emails': None,
                'topdir': '/mnt/koji',
                'topurl': None,
                'workdir': '/tmp/koji',
                'pluginpath': '/usr/lib/koji-builder-plugins',
                'mockdir': '/var/lib/mock',
                'mockuser': 'kojibuilder',
                'packager': 'Koji',
                'vendor': 'Koji',
                'distribution': 'Koji',
                'mockhost': 'koji-linux-gnu',
                'smtphost': 'example.com',
                'from_addr': 'Koji Build System <buildsys@example.com>',
                'krb_principal': None,
                'host_principal_format': 'compile/%s@EXAMPLE.COM',
                'keytab': '/etc/kojid/kojid.keytab',
                'ccache': '/var/tmp/kojid.ccache',
                'server': None,
                'user': None,
                'password': None,
                'retry_interval': 60,
                'max_retries': 120,
                'offline_retry': True,
                'offline_retry_interval': 120,
                'createrepo_skip_stat': True,
                'createrepo_update': True,
                'pkgurl': None,
                'allowed_scms': '',
                'cert': '/etc/kojid/client.crt',
                'ca': '/etc/kojid/clientca.crt',
                'serverca': '/etc/kojid/serverca.crt'}
    if config.has_section('kojid'):
        for name, value in config.items('kojid'):
            if name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval',
                        'max_retries', 'offline_retry_interval']:
                try:
                    defaults[name] = int(value)
                except ValueError:
                    quit("value for %s option must be a valid integer" % name)
            elif name in ['offline_retry', 'createrepo_skip_stat', 'createrepo_update']:
                defaults[name] = config.getboolean('kojid', name)
            elif name in ['plugin', 'plugins']:
                defaults['plugin'] = value.split()
            elif name in defaults.keys():
                defaults[name] = value
            else:
                quit("unknown config option: %s" % name)
    for name, value in defaults.items():
        if getattr(options, name, None) is None:
            setattr(options, name, value)

    #honor topdir
    if options.topdir:
        koji.BASEDIR = options.topdir
        koji.pathinfo.topdir = options.topdir

    #make sure workdir exists
    if not os.path.exists(options.workdir):
        koji.ensuredir(options.workdir)

    if not options.server:
        parser.error("--server argument required")

    if not options.pkgurl:
        parser.error("--pkgurl argument required")

def quit(msg=None, code=1):
    if msg:
        logging.getLogger("koji.build").error(msg)
        sys.stderr.write('%s\n' % msg)
        sys.stderr.flush()
    sys.exit(code)

if __name__ == "__main__":
    global options
    
    koji.add_file_logger("koji", "/var/log/kojid.log")
    #note we're setting logging params for all of koji*
    get_options()
    if options.debug:
        logging.getLogger("koji").setLevel(logging.DEBUG)
    elif options.verbose:
        logging.getLogger("koji").setLevel(logging.INFO)
    else:
        logging.getLogger("koji").setLevel(logging.WARN)
    if options.debug_task:
        logging.getLogger("koji.build.BaseTaskHandler").setLevel(logging.DEBUG)
    if options.admin_emails:
        koji.add_mail_logger("koji", options.admin_emails)

    #build session options
    session_opts = {}
    for k in ('user','password','debug_xmlrpc', 'debug',
              'retry_interval', 'max_retries', 'offline_retry', 'offline_retry_interval'):
        v = getattr(options, k, None)
        if v is not None:
            session_opts[k] = v
    #start a session and login
    session = koji.ClientSession(options.server, session_opts)
    if os.path.isfile(options.cert):
        try:
            # authenticate using SSL client certificates
            session.ssl_login(options.cert, options.ca,
                              options.serverca)
        except koji.AuthError, e:
            quit("Error: Unable to log in: %s" % e)
        except xmlrpclib.ProtocolError:
            quit("Error: Unable to connect to server %s" % (options.server))        
    elif options.user:
        try:
            # authenticate using user/password
            session.login()
        except koji.AuthError:
            quit("Error: Unable to log in. Bad credentials?")
        except xmlrpclib.ProtocolError:
            quit("Error: Unable to connect to server %s" % (options.server))
    elif sys.modules.has_key('krbV'):
        krb_principal = options.krb_principal
        if krb_principal is None:
            krb_principal = options.host_principal_format % socket.getfqdn()
        try:
            session.krb_login(principal=krb_principal,
                              keytab=options.keytab,
                              ccache=options.ccache)
        except krbV.Krb5Error, e:
            quit("Kerberos authentication failed: '%s' (%s)" % (e.message, e.err_code))
        except socket.error, e:
            quit("Could not connect to Kerberos authentication service: '%s'" % e.args[1])
    else:
        quit("No username/password supplied and Kerberos missing or not configured")
    #make session exclusive
    try:
        session.exclusiveSession(force=options.force_lock)
    except koji.AuthLockError:
        quit("Error: Unable to get lock. Trying using --force-lock")
    if not session.logged_in:
        quit("Error: Unknown login error")
    #make sure it works
    try:
        ret = session.echo("OK")
    except xmlrpclib.ProtocolError:
        quit("Error: Unable to connect to server %s" % (options.server))
    if ret != ["OK"]:
        quit("Error: incorrect server response: %r" % (ret))

    # run main
    if options.daemon:
        #detach
        koji.daemonize()
        main()
        # not reached
        assert False
    elif not options.skip_main:
        koji.add_stderr_logger("koji")
        main()