enable creation of LiveCD/DVD images in Koji

Signed-off-by: Mike Bonnet <mikeb@redhat.com>

builder/kojid

@@ -58,6 +58,11 @@ from gzip import GzipFile
 from optparse import OptionParser
 from StringIO import StringIO
 from xmlrpclib import Fault
+import pykickstart.parser as ksparser
+import pykickstart.version as ksversion
+import pykickstart.commands.repo as ksrepo
+import hashlib
+import iso9660 # from pycdio
 
 # our private modules
 sys.path.insert(0, '/usr/share/koji-builder/lib')
@@ -310,7 +315,7 @@ class BuildRoot(object):
         self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self)
         self.config = session.getBuildConfig(self.tag_id, event=self.event_id)
 
-    def _new(self, tag, arch, task_id, repo_id=None, install_group='build', setup_dns=False):
+    def _new(self, tag, arch, task_id, repo_id=None, install_group='build', setup_dns=False, bind_opts=None):
         """Create a brand new repo"""
         if not repo_id:
             raise koji.BuildrootError, "A repo id must be provided"
@@ -343,6 +348,7 @@ class BuildRoot(object):
         self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self)
         self.install_group = install_group
         self.setup_dns = setup_dns
+        self.bind_opts = bind_opts
         self._writeMockConfig()
 
     def _writeMockConfig(self):
@@ -362,6 +368,7 @@ class BuildRoot(object):
         opts['buildroot_id'] = self.id
         opts['use_host_resolv'] = self.setup_dns
         opts['install_group'] = self.install_group
+        opts['bind_opts'] = self.bind_opts
         output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts)
 
         #write config
@@ -1451,6 +1458,64 @@ class BaseTaskHandler(object):
         if os.path.isfile(filename) and os.stat(filename).st_size > 0:
             session.uploadWrapper(filename, self.getUploadDir(), remoteName)
 
+    def genImageManifest(self, image, manifile):
+        """
+        Using iso9660 from pycdio, get the file manifest of the given image,
+        and save it to the text file manifile.
+        """
+        fd = open(manifile, 'w')
+        if not fd:
+            raise koji.GenericError, \
+                'Unable to open manifest file (%s) for writing!' % manifile
+        iso = iso9660.ISO9660.IFS(source=image)
+        if not iso.is_open():
+            raise koji.GenericError, \
+                'Could not open %s as an ISO-9660 image!' % image
+
+        # image metadata
+        id = iso.get_application_id()
+        if id is not None: fd.write("Application ID: %s\n" % id)
+        id = iso.get_preparer_id()
+        if id is not None: fd.write("Preparer ID: %s\n" % id)
+        id = iso.get_publisher_id()
+        if id is not None: fd.write("Publisher ID: %s\n" % id)
+        id = iso.get_system_id()
+        if id is not None: fd.write("System ID: %s\n" % id)
+        id = iso.get_volume_id()
+        if id is not None: fd.write("Volume ID: %s\n" % id)
+        id = iso.get_volumeset_id()
+        if id is not None: fd.write("Volumeset ID: %s\n" % id)
+
+        fd.write('\nSize(bytes)  File Name\n')
+        manifest = self.listImageDir(iso, '/')
+        for a_file in manifest:
+            fd.write(a_file)
+        fd.close()
+        iso.close()
+
+    def listImageDir(self, iso, path):
+        """
+        Helper function called recursively by getImageManifest. Returns a
+        listing of files/directories at the given path in an iso image obj.
+        """
+        manifest = []
+        file_stats = iso.readdir(path)
+        for stat in file_stats:
+
+            filename = stat[0]
+            size = stat[2]
+            is_dir = stat[4]
+            if filename == "." or filename == "..": continue
+
+            if is_dir == 1:
+                manifest.append("%-10d   %s\n" % (size, os.path.join(path,
+                    iso9660.name_translate(filename))))
+            else:
+                 manifest.extend(self.listImageDir(iso,
+                     os.path.join(path, filename)))
+
+        return manifest
+
     def localPath(self, relpath):
         """Return a local path to a remote file.
 
@@ -2094,7 +2159,245 @@ class TagBuildTask(BaseTaskHandler):
             exctype, value = sys.exc_info()[:2]
             session.host.tagNotification(False, tag_id, fromtag, build_id, user_id, ignore_success, "%s: %s" % (exctype, value))
             raise e
-            
+
+# LiveCDTask begins with a mock chroot, and then installs livecd-tools into it
+# via the livecd-build group. livecd-creator is then executed in the chroot
+# to create the LiveCD image.
+#
+class LiveCDTask(BaseTaskHandler):
+
+    Methods = ['createLiveCD']
+    _taskWeight = 1.5
+
+    def handler(self, arch, target, ksfile, opts=None):
+
+        global options
+        target_info = session.getBuildTarget(target)
+        build_tag = target_info['build_tag']
+        repo_info = session.getRepo(build_tag)
+
+        # Here we configure mock to bind mount a set of /dev directories
+        # so livecd-creator can use them.
+        # We use mknod to create the loopN device files later, but if we
+        # wanted to have mock do that work, our bind_opts variable would have
+        # this additional dictionary:
+        #	{ 'files' : {
+        #		'/dev/loop0' : '/dev/loop0',
+        #		'/dev/loop1' : '/dev/loop1',
+        #		'/dev/loop2' : '/dev/loop2',
+        #		'/dev/loop3' : '/dev/loop3',
+        #		'/dev/loop4' : '/dev/loop4',
+        #		'/dev/loop5' : '/dev/loop5',
+        #		'/dev/loop6' : '/dev/loop6',
+        #		'/dev/loop7' : '/dev/loop7'}}
+        bind_opts = {'dirs' : {
+                         '/dev/pts' : '/dev/pts',
+                         '/dev/mapper' : '/dev/mapper',
+                         '/selinux' : '/selinux'}
+                     }
+        rootopts = {'install_group': 'livecd-build',
+                    'setup_dns': True,
+                    'repo_id': repo_info['id'],
+                    'bind_opts' : bind_opts}
+
+        broot = BuildRoot(build_tag, arch, self.id, **rootopts)
+
+        # create the mock chroot
+        self.logger.debug("Initializing buildroot")
+        broot.init()
+
+        # Note that if the KS file existed locally, then "ksfile" is a relative
+        # path to it in the /mnt/koji/work directory. If not, then it is still
+        # the parameter the user passed in initially, and we assume it is a
+        # relative path in a remote scm. The user should have passed in an scm
+        # url with --scmurl.
+        tmpchroot = '/tmp'
+        scmdir = os.path.join(broot.rootdir(), tmpchroot[1:])
+        koji.ensuredir(scmdir)
+        self.logger.debug("ksfile = %s" % ksfile)
+        if opts['scmurl']:
+            scm = SCM(opts['scmurl'])
+            scm.assert_allowed(options.allowed_scms)
+            logfile = os.path.join(self.workdir, 'checkout.log')
+            scmsrcdir = scm.checkout(scmdir, self.getUploadDir(), logfile)
+            kspath = os.path.join(scmsrcdir, ksfile)
+        else:
+            kspath = self.localPath("work/%s" % ksfile)
+
+        # XXX: If the ks file came from a local path and has %include
+        #      macros, livecd-creator will fail because the included
+        #      kickstarts were not copied into the chroot. For now we
+        #      require users to flatten their kickstart file if submitting
+        #      the task with a local path.
+        #
+        #      Note that if an SCM URL was used instead, %include macros
+        #      may not be a problem if the included kickstarts are present
+        #      in the repository we checked out.
+
+        # Now we do some kickstart manipulation. If the user passed in a repo
+        # url with --repo, then we substitute that in for the repo(s) specified
+        # in the kickstart file. If --repo wasn't specified, then we use the
+        # repo associated with the target passed in initially.
+        self.uploadFile(kspath) # upload the original ks file
+        version = ksversion.makeVersion()
+        ks = ksparser.KickstartParser(version)
+        try:
+            ks.readKickstart(kspath)
+        except IOError, (err, msg):
+            raise koji.LiveCDError("Failed to read kickstart file "
+                                   "'%s' : %s" % (ksfile, msg))
+        except kserrors.KickstartError, e:
+            raise koji.LiveCDError("Failed to parse kickstart file "
+                                    "'%s' : %s" % (ksfile, e))
+
+        ks.handler.repo.repoList = [] # delete whatever the ks file told us
+        if opts['repo']:
+            user_repos = opts['repo'].split(',')
+            index = 0
+            for user_repo in user_repos:
+                ks.handler.repo.repoList.append(ksrepo.F8_RepoData(
+                   baseurl=user_repo, name='koji_override_%s' % index))
+                index += 1
+        else:
+            if opts['topurl']:
+                topurl = opts['topurl']
+            else:
+                topurl = getattr(options, 'topurl')
+            if not topurl:
+                raise koji.GenericError, 'topurl needs to be defined!'
+            path_info = koji.PathInfo(topdir=topurl)
+            repopath = path_info.repo(repo_info['id'],
+                                     target_info['build_tag_name'])
+            baseurl = '%s/%s' % (repopath, arch)
+            self.logger.debug('BASEURL: %s' % baseurl)
+            ks.handler.repo.repoList.append(ksrepo.F8_RepoData(
+                baseurl=baseurl, name='koji'))
+
+        # Write out the new ks file. Note that things may not be in the same
+        # order and comments in the original ks file may be lost.
+        kskoji = os.path.join(tmpchroot, 'koji.ks')
+        kspath = os.path.join(broot.rootdir(), kskoji[1:])
+        outfile = open(kspath, 'w')
+        outfile.write(ks.handler.__str__())
+        outfile.close()
+
+        # put the new ksfile in the output directory
+        if not os.path.exists(kspath):
+            raise koji.LiveCDError, "KS file missing: %s" % kspath
+        self.uploadFile(kspath)
+
+        cachedir = os.path.join(tmpchroot, 'koji-livecd')   # arbitrary
+        livecd_log = os.path.join(tmpchroot, 'livecd.log')  # path in chroot
+
+        # Create the loopback devices we need
+        mknod_cmd = 'for i in `seq 0 7`; do mknod /dev/loop$i b 7 $i; done'
+        rv = broot.mock(['--chroot', mknod_cmd])
+        if rv:
+            broot.expire()
+            raise koji.LiveCDError, \
+                "Could not create loopback device files! %S" % rv
+
+        # Run livecd-creator
+        livecd_cmd = '/usr/bin/livecd-creator -c %s -d -v --logfile=%s --cache=%s' % (kskoji, livecd_log, cachedir)
+
+        # run the livecd-creator command
+        rv = broot.mock(['--chroot', livecd_cmd])
+        self.uploadFile(os.path.join(broot.rootdir(), livecd_log[1:]))
+        if rv:
+            broot.expire()
+            raise koji.LiveCDError, \
+                "livecd-creator command failed, see livecd.log or root.log"
+
+        # Find the resultant iso
+        files = os.listdir(broot.rootdir())
+        isofile = None
+        for afile in files:
+            if '.iso' in afile:
+                isofile = afile
+                break
+        if not isofile:
+            raise koji.LiveCDError, 'could not find iso file in chroot'
+
+        isosrc = os.path.join(broot.rootdir(), isofile)
+        try:
+            filesize = os.path.getsize(isosrc)
+        except OSError:
+            self.logger.error('Could not find the ISO. Did livecd-creator ' +
+                              'complete without errors?')
+            raise koji.LiveCDError, 'missing image file: %s' % isosrc
+
+        # if filesize is greater than a 32-bit signed integer's range, the
+        # python XMLRPC module will break.
+        if filesize > 2147483647:
+            filesize = str(filesize)
+
+        # copy the iso out of the chroot. If we were given an isoname, this is
+        # where the renaming happens.
+        self.logger.debug('uploading image: %s' % isosrc)
+        if opts['isoname']:
+            isofile = opts['isoname']
+            if not isofile.endswith('.iso'):
+                isofile += '.iso'
+        self.uploadFile(isosrc, remoteName=isofile)
+
+        # Generate the file manifest of the image
+        manifest = os.path.join(broot.resultdir(), 'manifest.log')
+        self.genImageManifest(isosrc, manifest)
+        self.uploadFile(manifest)
+
+        if not opts['scratch']:
+            # Read the rpm header information from the yum cache livecd-creator
+            # used. We assume it was empty to start.
+            #
+            # LiveCD creator would have thrown an error if no repo was
+            # specified, so we assume the ones we parse are ok.
+            repos = os.listdir(os.path.join(broot.rootdir(), cachedir[1:]))
+            if len(repos) == 0:
+                raise koji.LiveCDError, 'No repos found in yum cache!'
+
+            hdrlist = []
+            fields = ['name', 'version', 'release', 'epoch', 'arch', \
+                      'buildtime', 'sigmd5']
+            for repo in repos:
+                pkgpath = os.path.join(broot.rootdir(), cachedir[1:], repo,
+                                       'packages')
+                pkgs = os.listdir(pkgpath)
+                for pkg in pkgs:
+                    pkgfile = os.path.join(pkgpath, pkg)
+                    hdr = koji.get_header_fields(pkgfile, fields)
+                    hdr['size'] = os.path.getsize(pkgfile)
+                    hdr['payloadhash'] = koji.hex_string(hdr['sigmd5'])
+                    del hdr['sigmd5']
+                    hdrlist.append(hdr)
+
+            # get a unique hash of the image file
+            sha256sum = hashlib.sha256()
+            image_fo = file(isosrc, 'r')
+            while True:
+                data = image_fo.read(1048576)
+                sha256sum.update(data)
+                if not len(data):
+                    break
+
+            hash = sha256sum.hexdigest()
+            self.logger.debug('digest: %s' % hash)
+
+            # Import info about the image into the database, unless this is a
+            # scratch image.
+            broot.markExternalRPMs(hdrlist)
+            image_id = session.importImage(self.id, isofile, filesize,
+                                           'LiveCD ISO', hash, hdrlist)
+
+        broot.expire()
+        if opts['scratch']:
+            return 'Scratch image created: %s' % \
+                   os.path.join(koji.pathinfo.work(),
+                                koji.pathinfo.taskrelpath(self.id), isofile)
+        else:
+            return 'Created image: %s' % \
+                   os.path.join(koji.pathinfo.imageFinalPath(),
+                                koji.pathinfo.livecdRelPath(image_id), isofile)
+
 class BuildSRPMFromSCMTask(BaseTaskHandler):
 
     Methods = ['buildSRPMFromSCM']

cli/koji

@@ -3811,6 +3811,100 @@ def handle_remove_external_repo(options, session, args):
                 continue
             session.removeExternalRepoFromTag(tag, repo)
 
+# This handler is for spinning livecd images
+#
+def handle_spin_livecd(options, session, args):
+    """Create a live CD image given a kickstart file"""
+
+    # Usage & option parsing.
+    usage = _("usage: %prog spin-livecd [options] <arch> <target> " +
+              "<kickstart-file>")
+    usage += _("\n(Specify the --help global option for a list of other " +
+               "help options)")
+    parser = OptionParser(usage=usage)
+    parser.add_option("--nowait", action="store_true",
+        help=_("Don't wait on livecd creation"))
+    parser.add_option("--noprogress", action="store_true",
+        help=_("Do not display progress of the upload"))
+    parser.add_option("--background", action="store_true",
+        help=_("Run the livecd creation task at a lower priority"))
+    parser.add_option("--isoname",
+        help=_("Use a custom name for the iso file"))
+    parser.add_option("--topurl",
+        help=_("Specify a topurl, override that which is in kojid.conf"))
+    parser.add_option("--scmurl",
+        help=_("The SCM URL to the kickstart file"))
+    parser.add_option("--scratch", action="store_true",
+        help=_("Create a scratch LiveCD image."))
+    parser.add_option("--repo",
+        help=_("Specify a comma-separated list of repos that will override\n" +
+               "the repo used to install RPMs in the LiveCD image. The \n" +
+               "repo associated with the target is the default."))
+    (task_options, args) = parser.parse_args(args)
+
+    # Make sure the target and kickstart is specified.
+    if len(args) != 3:
+        parser.error(_("Three arguments are required: an architecture, " +
+                       "a build target, and a relative \npath to a " +
+                       "kickstart file ."))
+        assert False
+
+    activate_session(session)
+
+    # Set the task's priority. Users can only lower it with --background.
+    priority = None
+    if task_options.background:
+        # relative to koji.PRIO_DEFAULT; higher means a "lower" priority.
+        priority = 5
+    if _running_in_bg() or task_options.noprogress:
+        callback = None
+    else:
+        callback = _progress_callback
+
+    # Set the architecture
+    arch = koji.canonArch(args[0])
+
+    # We do some early sanity checking of the given target.
+    # Kojid gets these values again later on, but we check now as a convenience
+    # for the user.
+    target = args[1]
+    tmp_target = session.getBuildTarget(target)
+    if not tmp_target:
+        parser.error(_("Unknown build target: %s" % target))
+    dest_tag = session.getTag(tmp_target['dest_tag'])
+    if not dest_tag:
+        parser.error(_("Unknown destination tag: %s" %
+                       tmp_target['dest_tag_name']))
+
+    # Upload the KS file to the staging area.
+    # If it's a URL, it's kojid's job to go get it when it does the checkout.
+    ksfile = args[2]
+
+    if not task_options.scmurl:
+        serverdir = _unique_path('cli-livecd')
+        session.uploadWrapper(ksfile, serverdir, callback=callback)
+        ksfile = os.path.join(serverdir, os.path.basename(ksfile))
+        print
+
+    livecd_opts = {}
+    livecd_opts['scmurl'] = task_options.scmurl
+    livecd_opts['isoname'] = task_options.isoname
+    livecd_opts['scratch'] = task_options.scratch
+    livecd_opts['topurl'] = task_options.topurl
+    livecd_opts['repo'] = task_options.repo
+
+    # finally, create the task. Flow continues in kojihub::livecd.
+    task_id = session.livecd(arch, target, ksfile, opts=livecd_opts,
+                             priority=priority)
+
+    print "Created task:", task_id
+    print "Task info: %s/taskinfo?taskID=%s" % (options.weburl, task_id)
+    if _running_in_bg() or task_options.nowait:
+        return
+    else:
+        session.logout()
+        return watch_tasks(session,[task_id])
+
 def handle_free_task(options, session, args):
     "[admin] Free a task"
     usage = _("usage: %prog free-task [options] <task-id> [<task-id> ...]")

docs/schema.sql

@@ -6,8 +6,10 @@ DROP TABLE build_notifications;
 DROP TABLE log_messages;
 
 DROP TABLE buildroot_listing;
+DROP TABLE imageinfo_listing;
 
 DROP TABLE rpminfo;
+DROP TABLE imageinfo;
 
 DROP TABLE group_package_listing;
 DROP TABLE group_req_listing;
@@ -97,6 +99,7 @@ CREATE TABLE permissions (
 INSERT INTO permissions (name) VALUES ('admin');
 INSERT INTO permissions (name) VALUES ('build');
 INSERT INTO permissions (name) VALUES ('repo');
+INSERT INTO permissions (name) VALUES ('livecd');
 
 CREATE TABLE user_perms (
 	user_id INTEGER NOT NULL REFERENCES users(id),
@@ -421,6 +424,16 @@ CREATE TABLE buildroot (
 	dirtyness INTEGER
 ) WITHOUT OIDS;
 
+-- track spun images (livecds, installation, VMs...)
+CREATE TABLE imageinfo (
+	id SERIAL NOT NULL PRIMARY KEY,
+        task_id INTEGER NOT NULL REFERENCES task(id),
+	filename TEXT NOT NULL,
+	filesize BIGINT NOT NULL,
+	hash TEXT NOT NULL,
+	mediatype VARCHAR(16) NOT NULL
+) WITHOUT OIDS;
+
 -- this table associates tags with builds.  an entry here tags a package
 CREATE TABLE tag_listing (
 	build_id INTEGER NOT NULL REFERENCES build (id),
@@ -571,6 +584,13 @@ CREATE TABLE buildroot_listing (
 ) WITHOUT OIDS;
 CREATE INDEX buildroot_listing_rpms ON buildroot_listing(rpm_id);
 
+-- tracks the contents of an image
+CREATE TABLE imageinfo_listing (
+	rpm_id INTEGER NOT NULL REFERENCES rpminfo(id),
+	image_id INTEGER NOT NULL REFERENCES imageinfo(id),
+	UNIQUE (rpm_id, image_id)
+) WITHOUT OIDS;
+
 CREATE TABLE log_messages (
     id SERIAL NOT NULL PRIMARY KEY,
     message TEXT NOT NULL,

hub/kojihub.py

@@ -4569,6 +4569,80 @@ def rpmdiff(basepath, rpmlist):
             raise koji.BuildError, 'mismatch when analyzing %s, rpmdiff output was:\n%s' % \
                 (os.path.basename(first_rpm), output)
 
+def importImageInternal(task_id, filename, filesize, mediatype, hash, rpmlist):
+    """
+    Import image info and the listing into the database, and move an image
+    to the final resting place. The filesize may be reported as a string if it
+    exceeds the 32-bit signed integer limit. This function will convert it if
+    need be. Not called for scratch images.
+    """
+
+    #sanity checks
+    host = Host()
+    host.verify()
+    task = Task(task_id)
+    task.assertHost(host.id)
+
+    imageinfo = {}
+    imageinfo['id'] = _singleValue("""SELECT nextval('imageinfo_id_seq')""")
+    imageinfo['taskid'] = task_id
+    imageinfo['filename'] = filename
+    imageinfo['filesize'] = int(filesize)
+    imageinfo['mediatype'] = mediatype
+    imageinfo['hash'] = hash
+    q = """INSERT INTO imageinfo (id,task_id,filename,filesize,
+           mediatype,hash)
+           VALUES (%(id)i,%(taskid)i,%(filename)s,%(filesize)i,
+           %(mediatype)s,%(hash)s)
+        """
+    _dml(q, imageinfo)
+
+    q = """INSERT INTO imageinfo_listing (image_id,rpm_id)
+           VALUES (%(id)i,%(rpminfo)i)"""
+
+    rpm_ids = []
+    for an_rpm in rpmlist:
+        location = an_rpm.get('location')
+        if location:
+            data = add_external_rpm(an_rpm, location, strict=False)
+        else:
+            data = get_rpm(an_rpm, strict=True)
+        rpm_id = data['id']
+        rpm_ids.append(rpm_id)
+
+    for rpm_id in rpm_ids:
+        imageinfo['rpminfo'] = rpm_id
+        _dml(q, imageinfo)
+
+    return imageinfo['id']
+
+def moveImageResults(task_id, image_id):
+    """
+    Move the image file from the work/task directory into its more
+    permanent resting place. This shouldn't be called for scratch images.
+    """
+    source_path = os.path.join(koji.pathinfo.work(),
+                               koji.pathinfo.taskrelpath(task_id))
+    final_path = os.path.join(koji.pathinfo.imageFinalPath(),
+                              koji.pathinfo.livecdRelPath(image_id))
+    src_files = os.listdir(source_path)
+    if os.path.exists(final_path):
+        raise koji.GenericError("Error moving LiveCD image: the final " +
+                                "destination already exists!")
+    koji.ensuredir(final_path)
+
+    got_iso = False
+    for fname in src_files:
+        if '.iso' in fname: got_iso = True
+        os.rename(os.path.join(source_path, fname),
+                  os.path.join(final_path, fname))
+        os.symlink(os.path.join(final_path, fname),
+                   os.path.join(source_path, fname))
+
+    if not got_iso:
+        raise koji.GenericError(
+              "Could not move the iso to the final destination!")
+
 #
 # XMLRPC Methods
 #
@@ -4626,6 +4700,71 @@ class RootExports(object):
 
         return make_task('chainbuild',[srcs,target,opts],**taskOpts)
 
+    # Create the livecd task. Called from handle_spin_livecd in the client.
+    #
+    def livecd (self, arch, target, ksfile, opts=None, priority=None):
+        """
+        Create a live CD image using a kickstart file and group package list.
+        """
+
+        if not context.session.hasPerm('livecd'):
+            raise koji.ActionNotAllowed, \
+                'You must have the "livecd" permission to run this task!'
+
+        taskOpts = {}
+        taskOpts['arch'] = arch
+        if priority:
+            if priority < 0:
+                if not context.session.hasPerm('admin'):
+                    raise koji.ActionNotAllowed, \
+                               'only admins may create high-priority tasks'
+
+            taskOpts['priority'] = koji.PRIO_DEFAULT + priority
+
+        return make_task('createLiveCD', [arch, target, ksfile, opts],
+                         **taskOpts)
+
+    # Database access to get imageinfo values. Used in parts of kojiweb.
+    #
+    def getImageInfo(self, imageID=None, taskID=None):
+        """
+        Return the row from imageinfo given an image_id OR build_root_id.
+        It is an error if neither are specified, and image_id takes precedence.
+        Filesize will be reported as a string if it exceeds the 32-bit signed
+        integer limit.
+        """
+        tables = ['imageinfo']
+        fields = ['imageinfo.id', 'filename', 'filesize', 'mediatype',
+                  'imageinfo.task_id', 'buildroot.id', 'hash']
+        aliases = ['id', 'filename', 'filesize', 'mediatype', 'task_id',
+                 'br_id', 'hash']
+        joins = ['buildroot ON imageinfo.task_id = buildroot.task_id']
+        if imageID:
+            clauses = ['imageinfo.id = %(imageID)s']
+        elif taskID:
+            clauses = ['imageinfo.task_id = %(taskID)s']
+
+        query = QueryProcessor(columns=fields, tables=tables, clauses=clauses,
+                               values=locals(), joins=joins, aliases=aliases)
+        ret = query.executeOne()
+
+        # additional tweaking
+        if ret:
+            ret['path'] = os.path.join(koji.pathinfo.imageFinalPath(),
+                koji.pathinfo.livecdRelPath(ret['id']))
+            # Again we're covering for huge filesizes. XMLRPC will complain if
+            # numbers exceed signed 32-bit integer ranges.
+            if ret['filesize'] > 2147483647:
+                ret['filesize'] = str(ret['filesize'])
+        return ret
+
+    # Called from kojid::LiveCDTask
+    def importImage(self, task_id, filename, filesize, mediatype, hash, rpmlist):
+        image_id = importImageInternal(task_id, filename, filesize, mediatype,
+                                       hash, rpmlist)
+        moveImageResults(task_id, image_id)
+        return image_id
+
     def hello(self,*args):
         return "Hello World"
 
@@ -4846,7 +4985,10 @@ class RootExports(object):
                     stat_map = {}
                     for attr in dir(stat_info):
                         if attr.startswith('st_'):
-                            stat_map[attr] = getattr(stat_info, attr)
+                            if attr == 'st_size':
+                                stat_map[attr] = str(getattr(stat_info, attr))
+                            else:
+                                stat_map[attr] = getattr(stat_info, attr)
                     ret[filename] = stat_map
             else:
                 ret = output
@@ -5489,8 +5631,8 @@ class RootExports(object):
                 mapping[int(key)] = mapping[key]
         return readFullInheritance(tag,event,reverse,stops,jumps)
 
-    def listRPMs(self, buildID=None, buildrootID=None, componentBuildrootID=None, hostID=None, arches=None, queryOpts=None):
+    def listRPMs(self, buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, arches=None, queryOpts=None):
-        """List RPMS.  If buildID and/or buildrootID are specified,
+        """List RPMS.  If buildID, imageID and/or buildrootID are specified,
         restrict the list of RPMs to only those RPMs that are part of that
         build, or were built in that buildroot.  If componentBuildrootID is specified,
         restrict the list to only those RPMs that will get pulled into that buildroot
@@ -5541,6 +5683,12 @@ class RootExports(object):
             fields.append(('buildroot_listing.is_update', 'is_update'))
             joins.append('buildroot_listing ON rpminfo.id = buildroot_listing.rpm_id')
             clauses.append('buildroot_listing.buildroot_id = %(componentBuildrootID)i')
+
+        # image specific constraints
+        if imageID != None:
+           clauses.append('imageinfo_listing.image_id = %(imageID)s')
+           joins.append('imageinfo_listing ON rpminfo.id = imageinfo_listing.rpm_id')
+
         if hostID != None:
             joins.append('buildroot ON rpminfo.buildroot_id = buildroot.id')
             clauses.append('buildroot.host_id = %(hostID)i')

koji/__init__.py

@@ -277,6 +277,10 @@ class ServerOffline(GenericError):
     """Raised when the server is offline"""
     faultCode = 1014
 
+class LiveCDError(GenericError):
+    """Raised when LiveCD Image creation fails"""
+    faultCode = 1015
+
 class MultiCallInProgress(object):
     """
     Placeholder class to be returned by method calls when in the process of
@@ -1082,6 +1086,10 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts)
         'rpmbuild_timeout': 86400
     }
 
+    # bind_opts are used to mount parts (or all of) /dev if needed.
+    # See kojid::LiveCDTask for a look at this option in action.
+    bind_opts = opts.get('bind_opts')
+
     files = {}
     if opts.get('use_host_resolv', False) and os.path.exists('/etc/hosts'):
         # if we're setting up DNS,
@@ -1142,6 +1150,15 @@ baseurl=%(url)s
     for key, value in plugin_conf.iteritems():
         parts.append("config_opts['plugin_conf'][%r] = %r\n" % (key, value))
     parts.append("\n")
+
+    if bind_opts:
+        # This line is REQUIRED for mock to work if bind_opts defined.
+        parts.append("config_opts['internal_dev_setup'] = False\n")
+        for key in bind_opts.keys():
+            for mnt_src, mnt_dest in bind_opts.get(key).iteritems():
+                parts.append("config_opts['plugin_conf']['bind_mount_opts'][%r].append((%r, %r))\n" % (key, mnt_src, mnt_dest))
+        parts.append("\n")
+
     for key, value in macros.iteritems():
         parts.append("config_opts['macros'][%r] = %r\n" % (key, value))
     parts.append("\n")
@@ -1246,6 +1263,14 @@ class PathInfo(object):
         """Return the relative path for the task work directory"""
         return "tasks/%s/%s" % (task_id % 10000, task_id)
 
+    def livecdRelPath(self, image_id):
+        """Return the relative path for the livecd image directory"""
+        return os.path.join('livecd', str(image_id % 10000), str(image_id))
+
+    def imageFinalPath(self):
+        """Return the absolute path to where completed images can be found"""
+        return os.path.join(self.topdir, 'images')
+
     def work(self):
         """Return the work dir"""
         return self.topdir + '/work'

www/conf/kojiweb.conf

@@ -12,6 +12,7 @@ Alias /koji "/usr/share/koji-web/scripts/"
     PythonOption SiteName Koji
     PythonOption KojiHubURL http://hub.example.com/kojihub
     PythonOption KojiPackagesURL http://server.example.com/mnt/koji/packages
+    PythonOption KojiImagesURL http://server.example.com/mnt/koji/images
     PythonOption WebPrincipal koji/web@EXAMPLE.COM
     PythonOption WebKeytab /etc/httpd.keytab
     PythonOption WebCCache /var/tmp/kojiweb.ccache

www/kojiweb/index.py

@@ -362,9 +362,10 @@ _TASKS = ['build',
           'createrepo',
           'buildNotification',
           'tagNotification',
-          'dependantTask']
+          'dependantTask',
+          'createLiveCD']
 # Tasks that can exist without a parent
-_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'newRepo', 'tagBuild', 'tagNotification', 'waitrepo']
+_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'newRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'createLiveCD']
 # Tasks that can have children
 _PARENT_TASKS = ['build', 'chainbuild', 'newRepo']
 
@@ -509,6 +510,12 @@ def taskinfo(req, taskID):
     if task['method'] == 'buildArch':
         buildTag = server.getTag(params[1])
         values['buildTag'] = buildTag
+    elif task['method'] == 'createLiveCD':
+        # 'arch' is param[0], which is already mentioned later in the page.
+        values['target'] = params[1]
+        values['kickstart'] = os.path.basename(params[2])
+        values['opts'] = params[3]
+        values['image'] = server.getImageInfo(taskID=taskID)
     elif task['method'] == 'buildSRPMFromSCM':
         if len(params) > 1:
             buildTag = server.getTag(params[1])
@@ -564,6 +571,24 @@ def taskinfo(req, taskID):
     
     return _genHTML(req, 'taskinfo.chtml')
 
+def imageinfo(req, imageID):
+    """Do some prep work and generate the imageinfo page for kojiweb."""
+    server = _getServer(req)
+    values = _initValues(req, 'Image Information')
+    imageURL = req.get_options().get('KojiImagesURL', 'http://localhost/images')
+    values['image'] = server.getImageInfo(imageID=imageID)
+    urlrelpath = koji.pathinfo.livecdRelPath(values['image']['id'])
+    filelist = []
+    for ofile in os.listdir(values['image']['path']):
+        relpath = os.path.join(urlrelpath, ofile)
+        if relpath.endswith('.iso'):
+            values['imageURL'] = imageURL + '/' + relpath
+        else:
+            filelist.append(imageURL + '/' + relpath)
+
+    values['logs'] = filelist
+    return _genHTML(req, 'imageinfo.chtml')
+
 def taskstatus(req, taskID):
     server = _getServer(req)
 
@@ -614,8 +639,11 @@ def getfile(req, taskID, name, offset=None, size=None):
         req.headers_out['Content-Disposition'] = 'attachment; filename=%s' % name
     elif name.endswith('.log'):
         req.content_type = 'text/plain'
+    elif name.endswith('.iso'):
+        req.content_type = 'application/octetstream'
+        req.headers_out['Content-Disposition'] = 'attachment; filename=%s' % name
 
-    file_size = file_info['st_size']
+    file_size = int(file_info['st_size'])
     if offset is None:
         offset = 0
     else:
@@ -1370,26 +1398,51 @@ def buildrootinfo(req, buildrootID, builtStart=None, builtOrder=None, componentS
     
     return _genHTML(req, 'buildrootinfo.chtml')
 
-def rpmlist(req, buildrootID, type, start=None, order='nvr'):
+def rpmlist(req, type, buildrootID=None, imageID=None, start=None, order='nvr'):
+    """
+    rpmlist requires a buildrootID OR an imageID to be passed in. From one
+    of these values it will paginate a list of rpms included in the
+    corresponding object. (buildroot or image)
+    """
+
     values = _initValues(req, 'RPM List', 'hosts')
     server = _getServer(req)
 
-    buildrootID = int(buildrootID)
+    if buildrootID != None:
-    buildroot = server.getBuildroot(buildrootID)
+        buildrootID = int(buildrootID)
-    if buildroot == None:
+        buildroot = server.getBuildroot(buildrootID)
-        raise koji.GenericError, 'unknown buildroot ID: %i' % buildrootID
+        values['buildroot'] = buildroot
+        if buildroot == None:
+            raise koji.GenericError, 'unknown buildroot ID: %i' % buildrootID
 
-    rpms = None
+        rpms = None
-    if type == 'component':
+        if type == 'component':
-        rpms = kojiweb.util.paginateMethod(server, values, 'listRPMs', kw={'componentBuildrootID': buildroot['id']},
+            rpms = kojiweb.util.paginateMethod(server, values, 'listRPMs',
-                                           start=start, dataName='rpms', prefix='rpm', order=order)
+                   kw={'componentBuildrootID': buildroot['id']},
-    elif type == 'built':
+                   start=start, dataName='rpms', prefix='rpm', order=order)
-        rpms = kojiweb.util.paginateMethod(server, values, 'listRPMs', kw={'buildrootID': buildroot['id']},
+        elif type == 'built':
-                                           start=start, dataName='rpms', prefix='rpm', order=order)
+            rpms = kojiweb.util.paginateMethod(server, values, 'listRPMs',
+                   kw={'buildrootID': buildroot['id']},
+                   start=start, dataName='rpms', prefix='rpm', order=order)
+        else:
+            raise koji.GenericError, 'unrecognized type of rpmlist'
+
+    elif imageID != None:
+
+        values['image'] = server.getImageInfo(imageID=imageID)
+        # If/When future image types are supported, add elifs here if needed.
+        if type == 'image':
+            rpms =  kojiweb.util.paginateMethod(server, values, 'listRPMs',
+                    kw={'imageID': imageID}, \
+                    start=start, dataName='rpms', prefix='rpm', order=order)
+        else:
+            raise koji.GenericError, 'unrecognized type of image rpmlist'
+
+    else:
+        # It is an error if neither buildrootID and imageID are defined.
+        raise koji.GenericError, 'Both buildrootID and imageID are None'
 
-    values['buildroot'] = buildroot
     values['type'] = type
-
     values['order'] = order
 
     return _genHTML(req, 'rpmlist.chtml')

www/kojiweb/rpmlist.chtml

@@ -2,19 +2,39 @@
 
 #include "includes/header.chtml"
 
+#def getID()
+  #if $type == 'image'
+imageID=$image.id #slurp
+  #else
+buildrootID=$buildroot.id #slurp
+  #end if
+#end def
+
+#def getColspan()
+  #if $type == 'component'
+"colspan=3"
+  #elif $type == 'image'
+"colspan=2"
+  #else
+"colspan=1"
+  #end if
+#end def
+
   #if $type == 'component'
   <h4>Component RPMs of buildroot $buildroot.tag_name-$buildroot.id-$buildroot.repo_id</h4>
+  #elif $type == 'image'
+  <h4>RPMs installed in <a href="imageinfo?imageID=$image.id">$image.filename</a></h4>
   #else
   <h4>RPMs built in buildroot $buildroot.tag_name-$buildroot.id-$buildroot.repo_id</h4>
   #end if
 
   <table class="data-list">
     <tr>
-      <td class="paginate" colspan="#if $type == 'component' then '3' else '1'#">
+      <td class="paginate" $getColspan()>
         #if $len($rpmPages) > 1
         <form class="pageJump" action="">
           Page:
-          <select onchange="javascript: window.location = 'rpmlist?buildrootID=$buildroot.id&start=' + this.value * $rpmRange + '$util.passthrough($self, 'order', 'type')';">
+          <select onchange="javascript: window.location = 'rpmlist?$getID()&start=' + this.value * $rpmRange + '$util.passthrough($self, 'order', 'type')';">
             #for $pageNum in $rpmPages
             <option value="$pageNum"#if $pageNum == $rpmCurrentPage then ' selected="selected"' else ''#>#echo $pageNum + 1#</option>
             #end for
@@ -22,21 +42,23 @@
         </form>
         #end if
         #if $rpmStart > 0
-        <a href="rpmlist?buildrootID=$buildroot.id&start=#echo $rpmStart - $rpmRange #$util.passthrough($self, 'order', 'type')">&lt;&lt;&lt;</a>
+        <a href="rpmlist?$getID()&start=#echo $rpmStart - $rpmRange #$util.passthrough($self, 'order', 'type')">&lt;&lt;&lt;</a>
         #end if
         #if $totalRpms != 0
         <strong>RPMs #echo $rpmStart + 1 # through #echo $rpmStart + $rpmCount # of $totalRpms</strong>
         #end if
         #if $rpmStart + $rpmCount < $totalRpms
-        <a href="rpmlist?buildrootID=$buildroot.id&start=#echo $rpmStart + $rpmRange#$util.passthrough($self, 'order', 'type')">&gt;&gt;&gt;</a>
+        <a href="rpmlist?$getID()&start=#echo $rpmStart + $rpmRange#$util.passthrough($self, 'order', 'type')">&gt;&gt;&gt;</a>
         #end if
       </td>
     </tr>
     <tr class="list-header">
-      <th><a href="rpmlist?buildrootID=$buildroot.id&order=$util.toggleOrder($self, 'nvr')$util.passthrough($self, 'type')">NVR</a> $util.sortImage($self, 'nvr')</th>
+      <th><a href="rpmlist?$getID()&order=$util.toggleOrder($self, 'nvr')$util.passthrough($self, 'type')">NVR</a> $util.sortImage($self, 'nvr')</th>
       #if $type == 'component'
-      <th><a href="rpmlist?buildrootID=$buildroot.id&order=$util.toggleOrder($self, 'external_repo_name')$util.passthrough($self, 'type')">Origin</a> $util.sortImage($self, 'external_repo_name')</th>
+      <th><a href="rpmlist?$getID()&order=$util.toggleOrder($self, 'external_repo_name')$util.passthrough($self, 'type')">Origin</a> $util.sortImage($self, 'external_repo_name')</th>
-      <th><a href="rpmlist?buildrootID=$buildroot.id&order=$util.toggleOrder($self, 'is_update')$util.passthrough($self, 'type')">Update?</a> $util.sortImage($self, 'is_update')</th>
+      <th><a href="rpmlist?$getID()&order=$util.toggleOrder($self, 'is_update')$util.passthrough($self, 'type')">Update?</a> $util.sortImage($self, 'is_update')</th>
+      #elif $type == 'image'
+      <th><a href="rpmlist?$getID()&order=$util.toggleOrder($self, 'external_repo_name')$util.passthrough($self, 'type')">Origin</a> $util.sortImage($self, 'external_repo_name')</th>
       #end if
     </tr>
     #if $len($rpms) > 0
@@ -44,13 +66,11 @@
     <tr class="$util.rowToggle($self)">
       #set $epoch = ($rpm.epoch != None and $str($rpm.epoch) + ':' or '')
       <td><a href="rpminfo?rpmID=$rpm.id">$rpm.name-$epoch$rpm.version-$rpm.release.${rpm.arch}.rpm</a></td>
-      #if $type == 'component'
       #if $rpm.external_repo_id == 0
       <td>internal</td>
       #else
       <td><a href="externalrepoinfo?extrepoID=$rpm.external_repo_id">$rpm.external_repo_name</a></td>
       #end if
-      #end if
       #if $type == 'component'
       #set $update = $rpm.is_update and 'yes' or 'no'
       <td class="$update">$util.imageTag($update)</td>
@@ -67,7 +87,7 @@
         #if $len($rpmPages) > 1
         <form class="pageJump" action="">
           Page:
-          <select onchange="javascript: window.location = 'rpmlist?buildrootID=$buildroot.id&start=' + this.value * $rpmRange + '$util.passthrough($self, 'order', 'type')';">
+          <select onchange="javascript: window.location = 'rpmlist?$getID()&start=' + this.value * $rpmRange + '$util.passthrough($self, 'order', 'type')';">
             #for $pageNum in $rpmPages
             <option value="$pageNum"#if $pageNum == $rpmCurrentPage then ' selected="selected"' else ''#>#echo $pageNum + 1#</option>
             #end for
@@ -75,13 +95,13 @@
         </form>
         #end if
         #if $rpmStart > 0
-        <a href="rpmlist?buildrootID=$buildroot.id&start=#echo $rpmStart - $rpmRange #$util.passthrough($self, 'order', 'type')">&lt;&lt;&lt;</a>
+        <a href="rpmlist?$getID()&start=#echo $rpmStart - $rpmRange #$util.passthrough($self, 'order', 'type')">&lt;&lt;&lt;</a>
         #end if
         #if $totalRpms != 0
         <strong>RPMs #echo $rpmStart + 1 # through #echo $rpmStart + $rpmCount # of $totalRpms</strong>
         #end if
         #if $rpmStart + $rpmCount < $totalRpms
-        <a href="rpmlist?buildrootID=$buildroot.id&start=#echo $rpmStart + $rpmRange#$util.passthrough($self, 'order', 'type')">&gt;&gt;&gt;</a>
+        <a href="rpmlist?$getID()&start=#echo $rpmStart + $rpmRange#$util.passthrough($self, 'order', 'type')">&gt;&gt;&gt;</a>
         #end if
       </td>
     </tr>

www/kojiweb/taskinfo.chtml

@@ -116,6 +116,10 @@
         <strong>Source:</strong> $params[0]<br/>
         <strong>Build Target:</strong> <a href="buildtargetinfo?name=$params[1]">$params[1]</a><br/>
         $printOpts($params[2])
+        #elif $task.method == 'createLiveCD'
+        <strong>Target:</strong> $target<br/>
+        <strong>Kickstart File:</strong> $kickstart<br/>
+        $printOpts($opts)
         #elif $task.method == 'newRepo'
         <strong>Tag:</strong> <a href="taginfo?tagID=$tag.id">$tag.name</a><br/>
         #if $len($params) > 1
@@ -236,6 +240,18 @@
       </td>
     </tr>
     #end if
+    #if $task.method == 'createLiveCD'
+    <tr>
+      <th>Image Information</th>
+      <td>
+        #if $image
+        <a href="imageinfo?imageID=$image.id">$image.filename</a><br/>
+        #elif $opts.scratch
+        Scratch image, no information will be saved.<br/>
+        #end if
+      </td>
+    </tr>
+   #end if
     <tr>
       <th>Parent</th>
       <td>