From f2ea0f9bd0ab8182f474e978c5cac219636aba1b Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Thu, 20 Aug 2009 14:10:45 -0400 Subject: [PATCH 01/24] adding this join to the front of the list rather than the end improves the query plan and dramatically speeds up the query on postgres 8.1 (postgres 8.3 always uses the better plan) --- hub/kojihub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 36489de6..8d758857 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -2993,7 +2993,7 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, taskID=Non else: clauses.append('buildroot.state = %(state)i') if rpmID != None: - joins.append('buildroot_listing ON buildroot.id = buildroot_listing.buildroot_id') + joins.insert(0, 'buildroot_listing ON buildroot.id = buildroot_listing.buildroot_id') fields.append(('buildroot_listing.is_update', 'is_update')) clauses.append('buildroot_listing.rpm_id = %(rpmID)i') if taskID != None: From d93d05ab5f995947ca39a45dacb7b94da81751be Mon Sep 17 00:00:00 2001 From: Jay Greguske Date: Mon, 28 Sep 2009 14:07:01 -0400 Subject: [PATCH 02/24] enable creation of LiveCD/DVD images in Koji Signed-off-by: Mike Bonnet --- builder/kojid | 307 ++++++++++++++++++++++++++++++++++++- cli/koji | 94 ++++++++++++ docs/schema.sql | 20 +++ hub/kojihub.py | 154 ++++++++++++++++++- koji/__init__.py | 25 +++ www/conf/kojiweb.conf | 1 + www/kojiweb/index.py | 87 +++++++++-- www/kojiweb/rpmlist.chtml | 44 ++++-- www/kojiweb/taskinfo.chtml | 16 ++ 9 files changed, 714 insertions(+), 34 deletions(-) diff --git a/builder/kojid b/builder/kojid index 1809fd1b..96d73497 100755 --- a/builder/kojid +++ b/builder/kojid @@ -58,6 +58,11 @@ from gzip import GzipFile from optparse import OptionParser from StringIO import StringIO from xmlrpclib import Fault +import pykickstart.parser as ksparser +import pykickstart.version as ksversion +import pykickstart.commands.repo as ksrepo +import hashlib +import iso9660 # from pycdio # our private modules sys.path.insert(0, '/usr/share/koji-builder/lib') @@ -310,7 +315,7 @@ class BuildRoot(object): self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self) self.config = session.getBuildConfig(self.tag_id, event=self.event_id) - def _new(self, tag, arch, task_id, repo_id=None, install_group='build', setup_dns=False): + def _new(self, tag, arch, task_id, repo_id=None, install_group='build', setup_dns=False, bind_opts=None): """Create a brand new repo""" if not repo_id: raise koji.BuildrootError, "A repo id must be provided" @@ -343,6 +348,7 @@ class BuildRoot(object): self.name = "%(tag_name)s-%(id)s-%(repoid)s" % vars(self) self.install_group = install_group self.setup_dns = setup_dns + self.bind_opts = bind_opts self._writeMockConfig() def _writeMockConfig(self): @@ -362,6 +368,7 @@ class BuildRoot(object): opts['buildroot_id'] = self.id opts['use_host_resolv'] = self.setup_dns opts['install_group'] = self.install_group + opts['bind_opts'] = self.bind_opts output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts) #write config @@ -1451,6 +1458,64 @@ class BaseTaskHandler(object): if os.path.isfile(filename) and os.stat(filename).st_size > 0: session.uploadWrapper(filename, self.getUploadDir(), remoteName) + def genImageManifest(self, image, manifile): + """ + Using iso9660 from pycdio, get the file manifest of the given image, + and save it to the text file manifile. + """ + fd = open(manifile, 'w') + if not fd: + raise koji.GenericError, \ + 'Unable to open manifest file (%s) for writing!' % manifile + iso = iso9660.ISO9660.IFS(source=image) + if not iso.is_open(): + raise koji.GenericError, \ + 'Could not open %s as an ISO-9660 image!' % image + + # image metadata + id = iso.get_application_id() + if id is not None: fd.write("Application ID: %s\n" % id) + id = iso.get_preparer_id() + if id is not None: fd.write("Preparer ID: %s\n" % id) + id = iso.get_publisher_id() + if id is not None: fd.write("Publisher ID: %s\n" % id) + id = iso.get_system_id() + if id is not None: fd.write("System ID: %s\n" % id) + id = iso.get_volume_id() + if id is not None: fd.write("Volume ID: %s\n" % id) + id = iso.get_volumeset_id() + if id is not None: fd.write("Volumeset ID: %s\n" % id) + + fd.write('\nSize(bytes) File Name\n') + manifest = self.listImageDir(iso, '/') + for a_file in manifest: + fd.write(a_file) + fd.close() + iso.close() + + def listImageDir(self, iso, path): + """ + Helper function called recursively by getImageManifest. Returns a + listing of files/directories at the given path in an iso image obj. + """ + manifest = [] + file_stats = iso.readdir(path) + for stat in file_stats: + + filename = stat[0] + size = stat[2] + is_dir = stat[4] + if filename == "." or filename == "..": continue + + if is_dir == 1: + manifest.append("%-10d %s\n" % (size, os.path.join(path, + iso9660.name_translate(filename)))) + else: + manifest.extend(self.listImageDir(iso, + os.path.join(path, filename))) + + return manifest + def localPath(self, relpath): """Return a local path to a remote file. @@ -2094,7 +2159,245 @@ class TagBuildTask(BaseTaskHandler): exctype, value = sys.exc_info()[:2] session.host.tagNotification(False, tag_id, fromtag, build_id, user_id, ignore_success, "%s: %s" % (exctype, value)) raise e - + +# LiveCDTask begins with a mock chroot, and then installs livecd-tools into it +# via the livecd-build group. livecd-creator is then executed in the chroot +# to create the LiveCD image. +# +class LiveCDTask(BaseTaskHandler): + + Methods = ['createLiveCD'] + _taskWeight = 1.5 + + def handler(self, arch, target, ksfile, opts=None): + + global options + target_info = session.getBuildTarget(target) + build_tag = target_info['build_tag'] + repo_info = session.getRepo(build_tag) + + # Here we configure mock to bind mount a set of /dev directories + # so livecd-creator can use them. + # We use mknod to create the loopN device files later, but if we + # wanted to have mock do that work, our bind_opts variable would have + # this additional dictionary: + # { 'files' : { + # '/dev/loop0' : '/dev/loop0', + # '/dev/loop1' : '/dev/loop1', + # '/dev/loop2' : '/dev/loop2', + # '/dev/loop3' : '/dev/loop3', + # '/dev/loop4' : '/dev/loop4', + # '/dev/loop5' : '/dev/loop5', + # '/dev/loop6' : '/dev/loop6', + # '/dev/loop7' : '/dev/loop7'}} + bind_opts = {'dirs' : { + '/dev/pts' : '/dev/pts', + '/dev/mapper' : '/dev/mapper', + '/selinux' : '/selinux'} + } + rootopts = {'install_group': 'livecd-build', + 'setup_dns': True, + 'repo_id': repo_info['id'], + 'bind_opts' : bind_opts} + + broot = BuildRoot(build_tag, arch, self.id, **rootopts) + + # create the mock chroot + self.logger.debug("Initializing buildroot") + broot.init() + + # Note that if the KS file existed locally, then "ksfile" is a relative + # path to it in the /mnt/koji/work directory. If not, then it is still + # the parameter the user passed in initially, and we assume it is a + # relative path in a remote scm. The user should have passed in an scm + # url with --scmurl. + tmpchroot = '/tmp' + scmdir = os.path.join(broot.rootdir(), tmpchroot[1:]) + koji.ensuredir(scmdir) + self.logger.debug("ksfile = %s" % ksfile) + if opts['scmurl']: + scm = SCM(opts['scmurl']) + scm.assert_allowed(options.allowed_scms) + logfile = os.path.join(self.workdir, 'checkout.log') + scmsrcdir = scm.checkout(scmdir, self.getUploadDir(), logfile) + kspath = os.path.join(scmsrcdir, ksfile) + else: + kspath = self.localPath("work/%s" % ksfile) + + # XXX: If the ks file came from a local path and has %include + # macros, livecd-creator will fail because the included + # kickstarts were not copied into the chroot. For now we + # require users to flatten their kickstart file if submitting + # the task with a local path. + # + # Note that if an SCM URL was used instead, %include macros + # may not be a problem if the included kickstarts are present + # in the repository we checked out. + + # Now we do some kickstart manipulation. If the user passed in a repo + # url with --repo, then we substitute that in for the repo(s) specified + # in the kickstart file. If --repo wasn't specified, then we use the + # repo associated with the target passed in initially. + self.uploadFile(kspath) # upload the original ks file + version = ksversion.makeVersion() + ks = ksparser.KickstartParser(version) + try: + ks.readKickstart(kspath) + except IOError, (err, msg): + raise koji.LiveCDError("Failed to read kickstart file " + "'%s' : %s" % (ksfile, msg)) + except kserrors.KickstartError, e: + raise koji.LiveCDError("Failed to parse kickstart file " + "'%s' : %s" % (ksfile, e)) + + ks.handler.repo.repoList = [] # delete whatever the ks file told us + if opts['repo']: + user_repos = opts['repo'].split(',') + index = 0 + for user_repo in user_repos: + ks.handler.repo.repoList.append(ksrepo.F8_RepoData( + baseurl=user_repo, name='koji_override_%s' % index)) + index += 1 + else: + if opts['topurl']: + topurl = opts['topurl'] + else: + topurl = getattr(options, 'topurl') + if not topurl: + raise koji.GenericError, 'topurl needs to be defined!' + path_info = koji.PathInfo(topdir=topurl) + repopath = path_info.repo(repo_info['id'], + target_info['build_tag_name']) + baseurl = '%s/%s' % (repopath, arch) + self.logger.debug('BASEURL: %s' % baseurl) + ks.handler.repo.repoList.append(ksrepo.F8_RepoData( + baseurl=baseurl, name='koji')) + + # Write out the new ks file. Note that things may not be in the same + # order and comments in the original ks file may be lost. + kskoji = os.path.join(tmpchroot, 'koji.ks') + kspath = os.path.join(broot.rootdir(), kskoji[1:]) + outfile = open(kspath, 'w') + outfile.write(ks.handler.__str__()) + outfile.close() + + # put the new ksfile in the output directory + if not os.path.exists(kspath): + raise koji.LiveCDError, "KS file missing: %s" % kspath + self.uploadFile(kspath) + + cachedir = os.path.join(tmpchroot, 'koji-livecd') # arbitrary + livecd_log = os.path.join(tmpchroot, 'livecd.log') # path in chroot + + # Create the loopback devices we need + mknod_cmd = 'for i in `seq 0 7`; do mknod /dev/loop$i b 7 $i; done' + rv = broot.mock(['--chroot', mknod_cmd]) + if rv: + broot.expire() + raise koji.LiveCDError, \ + "Could not create loopback device files! %S" % rv + + # Run livecd-creator + livecd_cmd = '/usr/bin/livecd-creator -c %s -d -v --logfile=%s --cache=%s' % (kskoji, livecd_log, cachedir) + + # run the livecd-creator command + rv = broot.mock(['--chroot', livecd_cmd]) + self.uploadFile(os.path.join(broot.rootdir(), livecd_log[1:])) + if rv: + broot.expire() + raise koji.LiveCDError, \ + "livecd-creator command failed, see livecd.log or root.log" + + # Find the resultant iso + files = os.listdir(broot.rootdir()) + isofile = None + for afile in files: + if '.iso' in afile: + isofile = afile + break + if not isofile: + raise koji.LiveCDError, 'could not find iso file in chroot' + + isosrc = os.path.join(broot.rootdir(), isofile) + try: + filesize = os.path.getsize(isosrc) + except OSError: + self.logger.error('Could not find the ISO. Did livecd-creator ' + + 'complete without errors?') + raise koji.LiveCDError, 'missing image file: %s' % isosrc + + # if filesize is greater than a 32-bit signed integer's range, the + # python XMLRPC module will break. + if filesize > 2147483647: + filesize = str(filesize) + + # copy the iso out of the chroot. If we were given an isoname, this is + # where the renaming happens. + self.logger.debug('uploading image: %s' % isosrc) + if opts['isoname']: + isofile = opts['isoname'] + if not isofile.endswith('.iso'): + isofile += '.iso' + self.uploadFile(isosrc, remoteName=isofile) + + # Generate the file manifest of the image + manifest = os.path.join(broot.resultdir(), 'manifest.log') + self.genImageManifest(isosrc, manifest) + self.uploadFile(manifest) + + if not opts['scratch']: + # Read the rpm header information from the yum cache livecd-creator + # used. We assume it was empty to start. + # + # LiveCD creator would have thrown an error if no repo was + # specified, so we assume the ones we parse are ok. + repos = os.listdir(os.path.join(broot.rootdir(), cachedir[1:])) + if len(repos) == 0: + raise koji.LiveCDError, 'No repos found in yum cache!' + + hdrlist = [] + fields = ['name', 'version', 'release', 'epoch', 'arch', \ + 'buildtime', 'sigmd5'] + for repo in repos: + pkgpath = os.path.join(broot.rootdir(), cachedir[1:], repo, + 'packages') + pkgs = os.listdir(pkgpath) + for pkg in pkgs: + pkgfile = os.path.join(pkgpath, pkg) + hdr = koji.get_header_fields(pkgfile, fields) + hdr['size'] = os.path.getsize(pkgfile) + hdr['payloadhash'] = koji.hex_string(hdr['sigmd5']) + del hdr['sigmd5'] + hdrlist.append(hdr) + + # get a unique hash of the image file + sha256sum = hashlib.sha256() + image_fo = file(isosrc, 'r') + while True: + data = image_fo.read(1048576) + sha256sum.update(data) + if not len(data): + break + + hash = sha256sum.hexdigest() + self.logger.debug('digest: %s' % hash) + + # Import info about the image into the database, unless this is a + # scratch image. + broot.markExternalRPMs(hdrlist) + image_id = session.importImage(self.id, isofile, filesize, + 'LiveCD ISO', hash, hdrlist) + + broot.expire() + if opts['scratch']: + return 'Scratch image created: %s' % \ + os.path.join(koji.pathinfo.work(), + koji.pathinfo.taskrelpath(self.id), isofile) + else: + return 'Created image: %s' % \ + os.path.join(koji.pathinfo.imageFinalPath(), + koji.pathinfo.livecdRelPath(image_id), isofile) + class BuildSRPMFromSCMTask(BaseTaskHandler): Methods = ['buildSRPMFromSCM'] diff --git a/cli/koji b/cli/koji index 823a04e2..b0d14bbe 100755 --- a/cli/koji +++ b/cli/koji @@ -3811,6 +3811,100 @@ def handle_remove_external_repo(options, session, args): continue session.removeExternalRepoFromTag(tag, repo) +# This handler is for spinning livecd images +# +def handle_spin_livecd(options, session, args): + """Create a live CD image given a kickstart file""" + + # Usage & option parsing. + usage = _("usage: %prog spin-livecd [options] " + + "") + usage += _("\n(Specify the --help global option for a list of other " + + "help options)") + parser = OptionParser(usage=usage) + parser.add_option("--nowait", action="store_true", + help=_("Don't wait on livecd creation")) + parser.add_option("--noprogress", action="store_true", + help=_("Do not display progress of the upload")) + parser.add_option("--background", action="store_true", + help=_("Run the livecd creation task at a lower priority")) + parser.add_option("--isoname", + help=_("Use a custom name for the iso file")) + parser.add_option("--topurl", + help=_("Specify a topurl, override that which is in kojid.conf")) + parser.add_option("--scmurl", + help=_("The SCM URL to the kickstart file")) + parser.add_option("--scratch", action="store_true", + help=_("Create a scratch LiveCD image.")) + parser.add_option("--repo", + help=_("Specify a comma-separated list of repos that will override\n" + + "the repo used to install RPMs in the LiveCD image. The \n" + + "repo associated with the target is the default.")) + (task_options, args) = parser.parse_args(args) + + # Make sure the target and kickstart is specified. + if len(args) != 3: + parser.error(_("Three arguments are required: an architecture, " + + "a build target, and a relative \npath to a " + + "kickstart file .")) + assert False + + activate_session(session) + + # Set the task's priority. Users can only lower it with --background. + priority = None + if task_options.background: + # relative to koji.PRIO_DEFAULT; higher means a "lower" priority. + priority = 5 + if _running_in_bg() or task_options.noprogress: + callback = None + else: + callback = _progress_callback + + # Set the architecture + arch = koji.canonArch(args[0]) + + # We do some early sanity checking of the given target. + # Kojid gets these values again later on, but we check now as a convenience + # for the user. + target = args[1] + tmp_target = session.getBuildTarget(target) + if not tmp_target: + parser.error(_("Unknown build target: %s" % target)) + dest_tag = session.getTag(tmp_target['dest_tag']) + if not dest_tag: + parser.error(_("Unknown destination tag: %s" % + tmp_target['dest_tag_name'])) + + # Upload the KS file to the staging area. + # If it's a URL, it's kojid's job to go get it when it does the checkout. + ksfile = args[2] + + if not task_options.scmurl: + serverdir = _unique_path('cli-livecd') + session.uploadWrapper(ksfile, serverdir, callback=callback) + ksfile = os.path.join(serverdir, os.path.basename(ksfile)) + print + + livecd_opts = {} + livecd_opts['scmurl'] = task_options.scmurl + livecd_opts['isoname'] = task_options.isoname + livecd_opts['scratch'] = task_options.scratch + livecd_opts['topurl'] = task_options.topurl + livecd_opts['repo'] = task_options.repo + + # finally, create the task. Flow continues in kojihub::livecd. + task_id = session.livecd(arch, target, ksfile, opts=livecd_opts, + priority=priority) + + print "Created task:", task_id + print "Task info: %s/taskinfo?taskID=%s" % (options.weburl, task_id) + if _running_in_bg() or task_options.nowait: + return + else: + session.logout() + return watch_tasks(session,[task_id]) + def handle_free_task(options, session, args): "[admin] Free a task" usage = _("usage: %prog free-task [options] [ ...]") diff --git a/docs/schema.sql b/docs/schema.sql index a255b847..2c2eef33 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -6,8 +6,10 @@ DROP TABLE build_notifications; DROP TABLE log_messages; DROP TABLE buildroot_listing; +DROP TABLE imageinfo_listing; DROP TABLE rpminfo; +DROP TABLE imageinfo; DROP TABLE group_package_listing; DROP TABLE group_req_listing; @@ -97,6 +99,7 @@ CREATE TABLE permissions ( INSERT INTO permissions (name) VALUES ('admin'); INSERT INTO permissions (name) VALUES ('build'); INSERT INTO permissions (name) VALUES ('repo'); +INSERT INTO permissions (name) VALUES ('livecd'); CREATE TABLE user_perms ( user_id INTEGER NOT NULL REFERENCES users(id), @@ -421,6 +424,16 @@ CREATE TABLE buildroot ( dirtyness INTEGER ) WITHOUT OIDS; +-- track spun images (livecds, installation, VMs...) +CREATE TABLE imageinfo ( + id SERIAL NOT NULL PRIMARY KEY, + task_id INTEGER NOT NULL REFERENCES task(id), + filename TEXT NOT NULL, + filesize BIGINT NOT NULL, + hash TEXT NOT NULL, + mediatype VARCHAR(16) NOT NULL +) WITHOUT OIDS; + -- this table associates tags with builds. an entry here tags a package CREATE TABLE tag_listing ( build_id INTEGER NOT NULL REFERENCES build (id), @@ -571,6 +584,13 @@ CREATE TABLE buildroot_listing ( ) WITHOUT OIDS; CREATE INDEX buildroot_listing_rpms ON buildroot_listing(rpm_id); +-- tracks the contents of an image +CREATE TABLE imageinfo_listing ( + rpm_id INTEGER NOT NULL REFERENCES rpminfo(id), + image_id INTEGER NOT NULL REFERENCES imageinfo(id), + UNIQUE (rpm_id, image_id) +) WITHOUT OIDS; + CREATE TABLE log_messages ( id SERIAL NOT NULL PRIMARY KEY, message TEXT NOT NULL, diff --git a/hub/kojihub.py b/hub/kojihub.py index d56ce8bb..7238e353 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4569,6 +4569,80 @@ def rpmdiff(basepath, rpmlist): raise koji.BuildError, 'mismatch when analyzing %s, rpmdiff output was:\n%s' % \ (os.path.basename(first_rpm), output) +def importImageInternal(task_id, filename, filesize, mediatype, hash, rpmlist): + """ + Import image info and the listing into the database, and move an image + to the final resting place. The filesize may be reported as a string if it + exceeds the 32-bit signed integer limit. This function will convert it if + need be. Not called for scratch images. + """ + + #sanity checks + host = Host() + host.verify() + task = Task(task_id) + task.assertHost(host.id) + + imageinfo = {} + imageinfo['id'] = _singleValue("""SELECT nextval('imageinfo_id_seq')""") + imageinfo['taskid'] = task_id + imageinfo['filename'] = filename + imageinfo['filesize'] = int(filesize) + imageinfo['mediatype'] = mediatype + imageinfo['hash'] = hash + q = """INSERT INTO imageinfo (id,task_id,filename,filesize, + mediatype,hash) + VALUES (%(id)i,%(taskid)i,%(filename)s,%(filesize)i, + %(mediatype)s,%(hash)s) + """ + _dml(q, imageinfo) + + q = """INSERT INTO imageinfo_listing (image_id,rpm_id) + VALUES (%(id)i,%(rpminfo)i)""" + + rpm_ids = [] + for an_rpm in rpmlist: + location = an_rpm.get('location') + if location: + data = add_external_rpm(an_rpm, location, strict=False) + else: + data = get_rpm(an_rpm, strict=True) + rpm_id = data['id'] + rpm_ids.append(rpm_id) + + for rpm_id in rpm_ids: + imageinfo['rpminfo'] = rpm_id + _dml(q, imageinfo) + + return imageinfo['id'] + +def moveImageResults(task_id, image_id): + """ + Move the image file from the work/task directory into its more + permanent resting place. This shouldn't be called for scratch images. + """ + source_path = os.path.join(koji.pathinfo.work(), + koji.pathinfo.taskrelpath(task_id)) + final_path = os.path.join(koji.pathinfo.imageFinalPath(), + koji.pathinfo.livecdRelPath(image_id)) + src_files = os.listdir(source_path) + if os.path.exists(final_path): + raise koji.GenericError("Error moving LiveCD image: the final " + + "destination already exists!") + koji.ensuredir(final_path) + + got_iso = False + for fname in src_files: + if '.iso' in fname: got_iso = True + os.rename(os.path.join(source_path, fname), + os.path.join(final_path, fname)) + os.symlink(os.path.join(final_path, fname), + os.path.join(source_path, fname)) + + if not got_iso: + raise koji.GenericError( + "Could not move the iso to the final destination!") + # # XMLRPC Methods # @@ -4626,6 +4700,71 @@ class RootExports(object): return make_task('chainbuild',[srcs,target,opts],**taskOpts) + # Create the livecd task. Called from handle_spin_livecd in the client. + # + def livecd (self, arch, target, ksfile, opts=None, priority=None): + """ + Create a live CD image using a kickstart file and group package list. + """ + + if not context.session.hasPerm('livecd'): + raise koji.ActionNotAllowed, \ + 'You must have the "livecd" permission to run this task!' + + taskOpts = {} + taskOpts['arch'] = arch + if priority: + if priority < 0: + if not context.session.hasPerm('admin'): + raise koji.ActionNotAllowed, \ + 'only admins may create high-priority tasks' + + taskOpts['priority'] = koji.PRIO_DEFAULT + priority + + return make_task('createLiveCD', [arch, target, ksfile, opts], + **taskOpts) + + # Database access to get imageinfo values. Used in parts of kojiweb. + # + def getImageInfo(self, imageID=None, taskID=None): + """ + Return the row from imageinfo given an image_id OR build_root_id. + It is an error if neither are specified, and image_id takes precedence. + Filesize will be reported as a string if it exceeds the 32-bit signed + integer limit. + """ + tables = ['imageinfo'] + fields = ['imageinfo.id', 'filename', 'filesize', 'mediatype', + 'imageinfo.task_id', 'buildroot.id', 'hash'] + aliases = ['id', 'filename', 'filesize', 'mediatype', 'task_id', + 'br_id', 'hash'] + joins = ['buildroot ON imageinfo.task_id = buildroot.task_id'] + if imageID: + clauses = ['imageinfo.id = %(imageID)s'] + elif taskID: + clauses = ['imageinfo.task_id = %(taskID)s'] + + query = QueryProcessor(columns=fields, tables=tables, clauses=clauses, + values=locals(), joins=joins, aliases=aliases) + ret = query.executeOne() + + # additional tweaking + if ret: + ret['path'] = os.path.join(koji.pathinfo.imageFinalPath(), + koji.pathinfo.livecdRelPath(ret['id'])) + # Again we're covering for huge filesizes. XMLRPC will complain if + # numbers exceed signed 32-bit integer ranges. + if ret['filesize'] > 2147483647: + ret['filesize'] = str(ret['filesize']) + return ret + + # Called from kojid::LiveCDTask + def importImage(self, task_id, filename, filesize, mediatype, hash, rpmlist): + image_id = importImageInternal(task_id, filename, filesize, mediatype, + hash, rpmlist) + moveImageResults(task_id, image_id) + return image_id + def hello(self,*args): return "Hello World" @@ -4846,7 +4985,10 @@ class RootExports(object): stat_map = {} for attr in dir(stat_info): if attr.startswith('st_'): - stat_map[attr] = getattr(stat_info, attr) + if attr == 'st_size': + stat_map[attr] = str(getattr(stat_info, attr)) + else: + stat_map[attr] = getattr(stat_info, attr) ret[filename] = stat_map else: ret = output @@ -5489,8 +5631,8 @@ class RootExports(object): mapping[int(key)] = mapping[key] return readFullInheritance(tag,event,reverse,stops,jumps) - def listRPMs(self, buildID=None, buildrootID=None, componentBuildrootID=None, hostID=None, arches=None, queryOpts=None): - """List RPMS. If buildID and/or buildrootID are specified, + def listRPMs(self, buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, arches=None, queryOpts=None): + """List RPMS. If buildID, imageID and/or buildrootID are specified, restrict the list of RPMs to only those RPMs that are part of that build, or were built in that buildroot. If componentBuildrootID is specified, restrict the list to only those RPMs that will get pulled into that buildroot @@ -5541,6 +5683,12 @@ class RootExports(object): fields.append(('buildroot_listing.is_update', 'is_update')) joins.append('buildroot_listing ON rpminfo.id = buildroot_listing.rpm_id') clauses.append('buildroot_listing.buildroot_id = %(componentBuildrootID)i') + + # image specific constraints + if imageID != None: + clauses.append('imageinfo_listing.image_id = %(imageID)s') + joins.append('imageinfo_listing ON rpminfo.id = imageinfo_listing.rpm_id') + if hostID != None: joins.append('buildroot ON rpminfo.buildroot_id = buildroot.id') clauses.append('buildroot.host_id = %(hostID)i') diff --git a/koji/__init__.py b/koji/__init__.py index 94efcfb1..a04763f9 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -277,6 +277,10 @@ class ServerOffline(GenericError): """Raised when the server is offline""" faultCode = 1014 +class LiveCDError(GenericError): + """Raised when LiveCD Image creation fails""" + faultCode = 1015 + class MultiCallInProgress(object): """ Placeholder class to be returned by method calls when in the process of @@ -1082,6 +1086,10 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts) 'rpmbuild_timeout': 86400 } + # bind_opts are used to mount parts (or all of) /dev if needed. + # See kojid::LiveCDTask for a look at this option in action. + bind_opts = opts.get('bind_opts') + files = {} if opts.get('use_host_resolv', False) and os.path.exists('/etc/hosts'): # if we're setting up DNS, @@ -1142,6 +1150,15 @@ baseurl=%(url)s for key, value in plugin_conf.iteritems(): parts.append("config_opts['plugin_conf'][%r] = %r\n" % (key, value)) parts.append("\n") + + if bind_opts: + # This line is REQUIRED for mock to work if bind_opts defined. + parts.append("config_opts['internal_dev_setup'] = False\n") + for key in bind_opts.keys(): + for mnt_src, mnt_dest in bind_opts.get(key).iteritems(): + parts.append("config_opts['plugin_conf']['bind_mount_opts'][%r].append((%r, %r))\n" % (key, mnt_src, mnt_dest)) + parts.append("\n") + for key, value in macros.iteritems(): parts.append("config_opts['macros'][%r] = %r\n" % (key, value)) parts.append("\n") @@ -1246,6 +1263,14 @@ class PathInfo(object): """Return the relative path for the task work directory""" return "tasks/%s/%s" % (task_id % 10000, task_id) + def livecdRelPath(self, image_id): + """Return the relative path for the livecd image directory""" + return os.path.join('livecd', str(image_id % 10000), str(image_id)) + + def imageFinalPath(self): + """Return the absolute path to where completed images can be found""" + return os.path.join(self.topdir, 'images') + def work(self): """Return the work dir""" return self.topdir + '/work' diff --git a/www/conf/kojiweb.conf b/www/conf/kojiweb.conf index cff7dd69..17819adb 100644 --- a/www/conf/kojiweb.conf +++ b/www/conf/kojiweb.conf @@ -12,6 +12,7 @@ Alias /koji "/usr/share/koji-web/scripts/" PythonOption SiteName Koji PythonOption KojiHubURL http://hub.example.com/kojihub PythonOption KojiPackagesURL http://server.example.com/mnt/koji/packages + PythonOption KojiImagesURL http://server.example.com/mnt/koji/images PythonOption WebPrincipal koji/web@EXAMPLE.COM PythonOption WebKeytab /etc/httpd.keytab PythonOption WebCCache /var/tmp/kojiweb.ccache diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index ff54f037..88c3e125 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -362,9 +362,10 @@ _TASKS = ['build', 'createrepo', 'buildNotification', 'tagNotification', - 'dependantTask'] + 'dependantTask', + 'createLiveCD'] # Tasks that can exist without a parent -_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'newRepo', 'tagBuild', 'tagNotification', 'waitrepo'] +_TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'newRepo', 'tagBuild', 'tagNotification', 'waitrepo', 'createLiveCD'] # Tasks that can have children _PARENT_TASKS = ['build', 'chainbuild', 'newRepo'] @@ -509,6 +510,12 @@ def taskinfo(req, taskID): if task['method'] == 'buildArch': buildTag = server.getTag(params[1]) values['buildTag'] = buildTag + elif task['method'] == 'createLiveCD': + # 'arch' is param[0], which is already mentioned later in the page. + values['target'] = params[1] + values['kickstart'] = os.path.basename(params[2]) + values['opts'] = params[3] + values['image'] = server.getImageInfo(taskID=taskID) elif task['method'] == 'buildSRPMFromSCM': if len(params) > 1: buildTag = server.getTag(params[1]) @@ -564,6 +571,24 @@ def taskinfo(req, taskID): return _genHTML(req, 'taskinfo.chtml') +def imageinfo(req, imageID): + """Do some prep work and generate the imageinfo page for kojiweb.""" + server = _getServer(req) + values = _initValues(req, 'Image Information') + imageURL = req.get_options().get('KojiImagesURL', 'http://localhost/images') + values['image'] = server.getImageInfo(imageID=imageID) + urlrelpath = koji.pathinfo.livecdRelPath(values['image']['id']) + filelist = [] + for ofile in os.listdir(values['image']['path']): + relpath = os.path.join(urlrelpath, ofile) + if relpath.endswith('.iso'): + values['imageURL'] = imageURL + '/' + relpath + else: + filelist.append(imageURL + '/' + relpath) + + values['logs'] = filelist + return _genHTML(req, 'imageinfo.chtml') + def taskstatus(req, taskID): server = _getServer(req) @@ -614,8 +639,11 @@ def getfile(req, taskID, name, offset=None, size=None): req.headers_out['Content-Disposition'] = 'attachment; filename=%s' % name elif name.endswith('.log'): req.content_type = 'text/plain' + elif name.endswith('.iso'): + req.content_type = 'application/octetstream' + req.headers_out['Content-Disposition'] = 'attachment; filename=%s' % name - file_size = file_info['st_size'] + file_size = int(file_info['st_size']) if offset is None: offset = 0 else: @@ -1370,26 +1398,51 @@ def buildrootinfo(req, buildrootID, builtStart=None, builtOrder=None, componentS return _genHTML(req, 'buildrootinfo.chtml') -def rpmlist(req, buildrootID, type, start=None, order='nvr'): +def rpmlist(req, type, buildrootID=None, imageID=None, start=None, order='nvr'): + """ + rpmlist requires a buildrootID OR an imageID to be passed in. From one + of these values it will paginate a list of rpms included in the + corresponding object. (buildroot or image) + """ + values = _initValues(req, 'RPM List', 'hosts') server = _getServer(req) - buildrootID = int(buildrootID) - buildroot = server.getBuildroot(buildrootID) - if buildroot == None: - raise koji.GenericError, 'unknown buildroot ID: %i' % buildrootID + if buildrootID != None: + buildrootID = int(buildrootID) + buildroot = server.getBuildroot(buildrootID) + values['buildroot'] = buildroot + if buildroot == None: + raise koji.GenericError, 'unknown buildroot ID: %i' % buildrootID - rpms = None - if type == 'component': - rpms = kojiweb.util.paginateMethod(server, values, 'listRPMs', kw={'componentBuildrootID': buildroot['id']}, - start=start, dataName='rpms', prefix='rpm', order=order) - elif type == 'built': - rpms = kojiweb.util.paginateMethod(server, values, 'listRPMs', kw={'buildrootID': buildroot['id']}, - start=start, dataName='rpms', prefix='rpm', order=order) + rpms = None + if type == 'component': + rpms = kojiweb.util.paginateMethod(server, values, 'listRPMs', + kw={'componentBuildrootID': buildroot['id']}, + start=start, dataName='rpms', prefix='rpm', order=order) + elif type == 'built': + rpms = kojiweb.util.paginateMethod(server, values, 'listRPMs', + kw={'buildrootID': buildroot['id']}, + start=start, dataName='rpms', prefix='rpm', order=order) + else: + raise koji.GenericError, 'unrecognized type of rpmlist' + + elif imageID != None: + + values['image'] = server.getImageInfo(imageID=imageID) + # If/When future image types are supported, add elifs here if needed. + if type == 'image': + rpms = kojiweb.util.paginateMethod(server, values, 'listRPMs', + kw={'imageID': imageID}, \ + start=start, dataName='rpms', prefix='rpm', order=order) + else: + raise koji.GenericError, 'unrecognized type of image rpmlist' + + else: + # It is an error if neither buildrootID and imageID are defined. + raise koji.GenericError, 'Both buildrootID and imageID are None' - values['buildroot'] = buildroot values['type'] = type - values['order'] = order return _genHTML(req, 'rpmlist.chtml') diff --git a/www/kojiweb/rpmlist.chtml b/www/kojiweb/rpmlist.chtml index e4efcd0b..8636f353 100644 --- a/www/kojiweb/rpmlist.chtml +++ b/www/kojiweb/rpmlist.chtml @@ -2,19 +2,39 @@ #include "includes/header.chtml" +#def getID() + #if $type == 'image' +imageID=$image.id #slurp + #else +buildrootID=$buildroot.id #slurp + #end if +#end def + +#def getColspan() + #if $type == 'component' +"colspan=3" + #elif $type == 'image' +"colspan=2" + #else +"colspan=1" + #end if +#end def + #if $type == 'component'

Component RPMs of buildroot $buildroot.tag_name-$buildroot.id-$buildroot.repo_id

+ #elif $type == 'image' +

RPMs installed in $image.filename

#else

RPMs built in buildroot $buildroot.tag_name-$buildroot.id-$buildroot.repo_id

#end if - - + #if $type == 'component' - - + + + #elif $type == 'image' + #end if #if $len($rpms) > 0 @@ -44,13 +66,11 @@ #set $epoch = ($rpm.epoch != None and $str($rpm.epoch) + ':' or '') - #if $type == 'component' #if $rpm.external_repo_id == 0 #else #end if - #end if #if $type == 'component' #set $update = $rpm.is_update and 'yes' or 'no' @@ -67,7 +87,7 @@ #if $len($rpmPages) > 1 Page: - #for $pageNum in $rpmPages #end for @@ -75,13 +95,13 @@ #end if #if $rpmStart > 0 - <<< + <<< #end if #if $totalRpms != 0 RPMs #echo $rpmStart + 1 # through #echo $rpmStart + $rpmCount # of $totalRpms #end if #if $rpmStart + $rpmCount < $totalRpms - >>> + >>> #end if diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index effb52d3..0b15a576 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -116,6 +116,10 @@ Source: $params[0]
Build Target:$params[1]
$printOpts($params[2]) + #elif $task.method == 'createLiveCD' + Target: $target
+ Kickstart File: $kickstart
+ $printOpts($opts) #elif $task.method == 'newRepo' Tag:$tag.name
#if $len($params) > 1 @@ -236,6 +240,18 @@ #end if + #if $task.method == 'createLiveCD' + + + + + #end if #end if - #if $task.method == 'createLiveCD' + #if $task.method == 'createLiveCD' and $image - #end if + #end if
+ #if $len($rpmPages) > 1
Page: - #for $pageNum in $rpmPages #end for @@ -22,21 +42,23 @@
#end if #if $rpmStart > 0 - <<< + <<< #end if #if $totalRpms != 0 RPMs #echo $rpmStart + 1 # through #echo $rpmStart + $rpmCount # of $totalRpms #end if #if $rpmStart + $rpmCount < $totalRpms - >>> + >>> #end if
NVR $util.sortImage($self, 'nvr')NVR $util.sortImage($self, 'nvr')Origin $util.sortImage($self, 'external_repo_name')Update? $util.sortImage($self, 'is_update')Origin $util.sortImage($self, 'external_repo_name')Update? $util.sortImage($self, 'is_update')Origin $util.sortImage($self, 'external_repo_name')
$rpm.name-$epoch$rpm.version-$rpm.release.${rpm.arch}.rpminternal$rpm.external_repo_name$util.imageTag($update)
Image Information + #if $image + $image.filename
+ #elif $opts.scratch + Scratch image, no information will be saved.
+ #end if +
Parent From 66a86887b1deeed6d89a081f078b2d29796b3745 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Mon, 28 Sep 2009 14:09:13 -0400 Subject: [PATCH 03/24] mark spin-livecd as an admin task --- cli/koji | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/koji b/cli/koji index b0d14bbe..b3cc5bc9 100755 --- a/cli/koji +++ b/cli/koji @@ -3814,7 +3814,7 @@ def handle_remove_external_repo(options, session, args): # This handler is for spinning livecd images # def handle_spin_livecd(options, session, args): - """Create a live CD image given a kickstart file""" + """[admin] Create a live CD image given a kickstart file""" # Usage & option parsing. usage = _("usage: %prog spin-livecd [options] " + From f47af01a40b7baffff327961f49c59cfa8988aed Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Mon, 28 Sep 2009 14:23:46 -0400 Subject: [PATCH 04/24] specify the LiveCD target first, for consistency with the build command --- cli/koji | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/koji b/cli/koji index b3cc5bc9..7005eff8 100755 --- a/cli/koji +++ b/cli/koji @@ -3817,7 +3817,7 @@ def handle_spin_livecd(options, session, args): """[admin] Create a live CD image given a kickstart file""" # Usage & option parsing. - usage = _("usage: %prog spin-livecd [options] " + + usage = _("usage: %prog spin-livecd [options] " + "") usage += _("\n(Specify the --help global option for a list of other " + "help options)") @@ -3861,13 +3861,10 @@ def handle_spin_livecd(options, session, args): else: callback = _progress_callback - # Set the architecture - arch = koji.canonArch(args[0]) - # We do some early sanity checking of the given target. # Kojid gets these values again later on, but we check now as a convenience # for the user. - target = args[1] + target = args[0] tmp_target = session.getBuildTarget(target) if not tmp_target: parser.error(_("Unknown build target: %s" % target)) @@ -3876,6 +3873,9 @@ def handle_spin_livecd(options, session, args): parser.error(_("Unknown destination tag: %s" % tmp_target['dest_tag_name'])) + # Set the architecture + arch = koji.canonArch(args[1]) + # Upload the KS file to the staging area. # If it's a URL, it's kojid's job to go get it when it does the checkout. ksfile = args[2] From 78e9e65ddf7b3a00f65650f05a95cf8734cb85a3 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Mon, 28 Sep 2009 14:32:19 -0400 Subject: [PATCH 05/24] remove the topurl option to spin-livecd --- builder/kojid | 13 +++++-------- cli/koji | 3 --- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/builder/kojid b/builder/kojid index 96d73497..0de88679 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2259,26 +2259,23 @@ class LiveCDTask(BaseTaskHandler): baseurl=user_repo, name='koji_override_%s' % index)) index += 1 else: - if opts['topurl']: - topurl = opts['topurl'] - else: - topurl = getattr(options, 'topurl') + topurl = getattr(options, 'topurl') if not topurl: - raise koji.GenericError, 'topurl needs to be defined!' + raise koji.LiveCDError, 'topurl must be defined in kojid.conf' path_info = koji.PathInfo(topdir=topurl) repopath = path_info.repo(repo_info['id'], - target_info['build_tag_name']) + target_info['build_tag_name']) baseurl = '%s/%s' % (repopath, arch) self.logger.debug('BASEURL: %s' % baseurl) ks.handler.repo.repoList.append(ksrepo.F8_RepoData( - baseurl=baseurl, name='koji')) + baseurl=baseurl, name='koji')) # Write out the new ks file. Note that things may not be in the same # order and comments in the original ks file may be lost. kskoji = os.path.join(tmpchroot, 'koji.ks') kspath = os.path.join(broot.rootdir(), kskoji[1:]) outfile = open(kspath, 'w') - outfile.write(ks.handler.__str__()) + outfile.write(str(ks.handler)) outfile.close() # put the new ksfile in the output directory diff --git a/cli/koji b/cli/koji index 7005eff8..00bca1e4 100755 --- a/cli/koji +++ b/cli/koji @@ -3830,8 +3830,6 @@ def handle_spin_livecd(options, session, args): help=_("Run the livecd creation task at a lower priority")) parser.add_option("--isoname", help=_("Use a custom name for the iso file")) - parser.add_option("--topurl", - help=_("Specify a topurl, override that which is in kojid.conf")) parser.add_option("--scmurl", help=_("The SCM URL to the kickstart file")) parser.add_option("--scratch", action="store_true", @@ -3890,7 +3888,6 @@ def handle_spin_livecd(options, session, args): livecd_opts['scmurl'] = task_options.scmurl livecd_opts['isoname'] = task_options.isoname livecd_opts['scratch'] = task_options.scratch - livecd_opts['topurl'] = task_options.topurl livecd_opts['repo'] = task_options.repo # finally, create the task. Flow continues in kojihub::livecd. From e04395e7f3eb2b199d57f6f1de9a693e30fd68fb Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Mon, 28 Sep 2009 14:42:18 -0400 Subject: [PATCH 06/24] use assertPerm() so admins will inherit the permission --- hub/kojihub.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 7238e353..154e0d9e 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4707,9 +4707,7 @@ class RootExports(object): Create a live CD image using a kickstart file and group package list. """ - if not context.session.hasPerm('livecd'): - raise koji.ActionNotAllowed, \ - 'You must have the "livecd" permission to run this task!' + context.session.assertPerm('livecd') taskOpts = {} taskOpts['arch'] = arch From 445edc2018bdc76121f4b005028eacc89ce3a82f Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Mon, 28 Sep 2009 14:53:44 -0400 Subject: [PATCH 07/24] put createLiveCD tasks in their own channel --- docs/schema.sql | 1 + hub/kojihub.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/schema.sql b/docs/schema.sql index 2c2eef33..d9e1932b 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -170,6 +170,7 @@ CREATE TABLE channels ( -- create default channel INSERT INTO channels (name) VALUES ('default'); INSERT INTO channels (name) VALUES ('createrepo'); +INSERT INTO channels (name) VALUES ('livecd'); -- Here we track the build machines -- each host has an entry in the users table also diff --git a/hub/kojihub.py b/hub/kojihub.py index 154e0d9e..5dda9edf 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4709,7 +4709,7 @@ class RootExports(object): context.session.assertPerm('livecd') - taskOpts = {} + taskOpts = {'channel': 'livecd'} taskOpts['arch'] = arch if priority: if priority < 0: From bb164493dd1fccd33d053f9861553f092323e96e Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Mon, 28 Sep 2009 14:56:40 -0400 Subject: [PATCH 08/24] missed this one in the initial LiveCD commit --- www/kojiweb/imageinfo.chtml | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 www/kojiweb/imageinfo.chtml diff --git a/www/kojiweb/imageinfo.chtml b/www/kojiweb/imageinfo.chtml new file mode 100644 index 00000000..d2d77c73 --- /dev/null +++ b/www/kojiweb/imageinfo.chtml @@ -0,0 +1,62 @@ +#import koji +#import koji.util +#from os.path import basename +#from kojiweb import util + +#include "includes/header.chtml" + +

Information for image $image.filename

+ + + + + + + + + + + + + + + + + + + #if $len($image.hash) == 32 + + #elif $len($image.hash) == 40 + + #elif $len($image.hash) == 64 + + #elif $len($image.hash) == 96 + + #elif $len($image.hash) == 128 + + #else + + #end if + + + + + + + + + + + + + + + + +
Image ID$image.id
Build Root ID$image.br_id
File Name$image.filename
File Size$image.filesize bytes
Media Type$image.mediatype
Hash (MD5)$image.hashHash (SHA1)$image.hashHash (SHA256)$image.hashHash (SHA384)$image.hashHash (SHA512)$image.hashHash $image.hash
Task ID$image.task_id
Output
+ #for $ofile in $logs +     $basename($ofile)
+ #end for +
Included RPMs
Download Image
+ +#include "includes/footer.chtml" From baac7cc0da5dda0c4998df413121551fea329c79 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Mon, 28 Sep 2009 15:55:22 -0400 Subject: [PATCH 09/24] add extended info to createLiveCD task labels --- koji/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/koji/__init__.py b/koji/__init__.py index a04763f9..42e7396d 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1830,6 +1830,10 @@ def taskLabel(taskInfo): nvrs = taskInfo['request'][2] if isinstance(nvrs, list): extra += ', ' + ', '.join(nvrs) + elif method == 'createLiveCD': + if taskInfo.has_key('request'): + arch, target, ksfile = taskInfo['request'][:3] + extra = '%s, %s, %s' % (target, arch, os.path.basename(ksfile)) if extra: return '%s (%s)' % (method, extra) From 33dd85fcd9ddd1028c4757f558f9302225b9f538 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Mon, 28 Sep 2009 18:22:37 -0400 Subject: [PATCH 10/24] show the "Watch logs" link on the taskinfo page for createLiveCD tasks as well --- www/kojiweb/taskinfo.chtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 0b15a576..226698e1 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -310,7 +310,7 @@ ${excClass.__name__}: $cgi.escape($str($result))
#end for #if $task.state not in ($koji.TASK_STATES.CLOSED, $koji.TASK_STATES.CANCELED, $koji.TASK_STATES.FAILED) and \ - $task.method in ('buildSRPMFromSCM', 'buildArch', 'createrepo') + $task.method in ('buildSRPMFromSCM', 'buildArch', 'createLiveCD', 'createrepo')
Watch logs #end if From f39637bf2a5f31715d84cfcbee8c82d0b7de3da9 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Tue, 29 Sep 2009 11:12:51 -0400 Subject: [PATCH 11/24] only pass non-null options to createLiveCD --- builder/kojid | 12 +++++++----- cli/koji | 7 +++---- www/kojiweb/index.py | 6 +----- www/kojiweb/taskinfo.chtml | 17 ++++++++--------- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/builder/kojid b/builder/kojid index 0de88679..f9389e54 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2170,6 +2170,8 @@ class LiveCDTask(BaseTaskHandler): _taskWeight = 1.5 def handler(self, arch, target, ksfile, opts=None): + if not opts: + opts = {} global options target_info = session.getBuildTarget(target) @@ -2215,7 +2217,7 @@ class LiveCDTask(BaseTaskHandler): scmdir = os.path.join(broot.rootdir(), tmpchroot[1:]) koji.ensuredir(scmdir) self.logger.debug("ksfile = %s" % ksfile) - if opts['scmurl']: + if opts.get('scmurl'): scm = SCM(opts['scmurl']) scm.assert_allowed(options.allowed_scms) logfile = os.path.join(self.workdir, 'checkout.log') @@ -2251,7 +2253,7 @@ class LiveCDTask(BaseTaskHandler): "'%s' : %s" % (ksfile, e)) ks.handler.repo.repoList = [] # delete whatever the ks file told us - if opts['repo']: + if opts.get('repo'): user_repos = opts['repo'].split(',') index = 0 for user_repo in user_repos: @@ -2331,7 +2333,7 @@ class LiveCDTask(BaseTaskHandler): # copy the iso out of the chroot. If we were given an isoname, this is # where the renaming happens. self.logger.debug('uploading image: %s' % isosrc) - if opts['isoname']: + if opts.get('isoname'): isofile = opts['isoname'] if not isofile.endswith('.iso'): isofile += '.iso' @@ -2342,7 +2344,7 @@ class LiveCDTask(BaseTaskHandler): self.genImageManifest(isosrc, manifest) self.uploadFile(manifest) - if not opts['scratch']: + if not opts.get('scratch'): # Read the rpm header information from the yum cache livecd-creator # used. We assume it was empty to start. # @@ -2386,7 +2388,7 @@ class LiveCDTask(BaseTaskHandler): 'LiveCD ISO', hash, hdrlist) broot.expire() - if opts['scratch']: + if opts.get('scratch'): return 'Scratch image created: %s' % \ os.path.join(koji.pathinfo.work(), koji.pathinfo.taskrelpath(self.id), isofile) diff --git a/cli/koji b/cli/koji index 00bca1e4..4f5a150d 100755 --- a/cli/koji +++ b/cli/koji @@ -3885,10 +3885,9 @@ def handle_spin_livecd(options, session, args): print livecd_opts = {} - livecd_opts['scmurl'] = task_options.scmurl - livecd_opts['isoname'] = task_options.isoname - livecd_opts['scratch'] = task_options.scratch - livecd_opts['repo'] = task_options.repo + for opt in ['scratch', 'scmurl', 'isoname', 'repo']: + if getattr(task_options, opt): + livecd_opts[opt] = getattr(task_options, opt) # finally, create the task. Flow continues in kojihub::livecd. task_id = session.livecd(arch, target, ksfile, opts=livecd_opts, diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 88c3e125..107dd77e 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -511,10 +511,6 @@ def taskinfo(req, taskID): buildTag = server.getTag(params[1]) values['buildTag'] = buildTag elif task['method'] == 'createLiveCD': - # 'arch' is param[0], which is already mentioned later in the page. - values['target'] = params[1] - values['kickstart'] = os.path.basename(params[2]) - values['opts'] = params[3] values['image'] = server.getImageInfo(taskID=taskID) elif task['method'] == 'buildSRPMFromSCM': if len(params) > 1: @@ -599,7 +595,7 @@ def taskstatus(req, taskID): files = server.listTaskOutput(taskID, stat=True) output = '%i:%s\n' % (task['id'], koji.TASK_STATES[task['state']]) for filename, file_stats in files.items(): - output += '%s:%i\n' % (filename, file_stats['st_size']) + output += '%s:%s\n' % (filename, file_stats['st_size']) return output diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 226698e1..d0e4361d 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -117,9 +117,12 @@ Build Target: $params[1]
$printOpts($params[2]) #elif $task.method == 'createLiveCD' - Target: $target
- Kickstart File: $kickstart
- $printOpts($opts) + Arch: $params[0]
+ Target: $params[1]
+ Kickstart File: $params[2]
+ #if $len($params) > 3 + $printOpts($params[3]) + #end if #elif $task.method == 'newRepo' Tag: $tag.name
#if $len($params) > 1 @@ -240,18 +243,14 @@
Image Information - #if $image $image.filename
- #elif $opts.scratch - Scratch image, no information will be saved.
- #end if
Parent From 7d954b8b22097895a4372aa087a728250b256587 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Tue, 29 Sep 2009 13:23:20 -0400 Subject: [PATCH 12/24] minor tweaks to the web UI for consistency --- hub/kojihub.py | 13 ++++++------- www/kojiweb/imageinfo.chtml | 28 +++++++++++++--------------- www/kojiweb/index.py | 11 ++++++++--- www/kojiweb/taskinfo.chtml | 2 +- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 5dda9edf..5e02466f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4735,12 +4735,12 @@ class RootExports(object): fields = ['imageinfo.id', 'filename', 'filesize', 'mediatype', 'imageinfo.task_id', 'buildroot.id', 'hash'] aliases = ['id', 'filename', 'filesize', 'mediatype', 'task_id', - 'br_id', 'hash'] + 'br_id', 'hash'] joins = ['buildroot ON imageinfo.task_id = buildroot.task_id'] if imageID: - clauses = ['imageinfo.id = %(imageID)s'] + clauses = ['imageinfo.id = %(imageID)i'] elif taskID: - clauses = ['imageinfo.task_id = %(taskID)s'] + clauses = ['imageinfo.task_id = %(taskID)i'] query = QueryProcessor(columns=fields, tables=tables, clauses=clauses, values=locals(), joins=joins, aliases=aliases) @@ -4750,10 +4750,9 @@ class RootExports(object): if ret: ret['path'] = os.path.join(koji.pathinfo.imageFinalPath(), koji.pathinfo.livecdRelPath(ret['id'])) - # Again we're covering for huge filesizes. XMLRPC will complain if - # numbers exceed signed 32-bit integer ranges. - if ret['filesize'] > 2147483647: - ret['filesize'] = str(ret['filesize']) + # Always return filesize as a string instead of an int so XMLRPC doesn't + # complain about 32-bit overflow + ret['filesize'] = str(ret['filesize']) return ret # Called from kojid::LiveCDTask diff --git a/www/kojiweb/imageinfo.chtml b/www/kojiweb/imageinfo.chtml index d2d77c73..f8ebd26b 100644 --- a/www/kojiweb/imageinfo.chtml +++ b/www/kojiweb/imageinfo.chtml @@ -9,45 +9,43 @@ - - - - + - - + + #if $len($image.hash) == 32 - + #elif $len($image.hash) == 40 - + #elif $len($image.hash) == 64 - + #elif $len($image.hash) == 96 - + #elif $len($image.hash) == 128 - + #else #end if - + + + + - - diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 107dd77e..9a1871b2 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -572,8 +572,13 @@ def imageinfo(req, imageID): server = _getServer(req) values = _initValues(req, 'Image Information') imageURL = req.get_options().get('KojiImagesURL', 'http://localhost/images') - values['image'] = server.getImageInfo(imageID=imageID) - urlrelpath = koji.pathinfo.livecdRelPath(values['image']['id']) + imageID = int(imageID) + image = server.getImageInfo(imageID=imageID) + values['image'] = image + values['title'] = image['filename'] + ' | Image Information' + urlrelpath = koji.pathinfo.livecdRelPath(image['id']) + values['buildroot'] = server.getBuildroot(image['br_id']) + values['task'] = server.getTaskInfo(image['task_id'], request=True) filelist = [] for ofile in os.listdir(values['image']['path']): relpath = os.path.join(urlrelpath, ofile) @@ -1424,7 +1429,7 @@ def rpmlist(req, type, buildrootID=None, imageID=None, start=None, order='nvr'): raise koji.GenericError, 'unrecognized type of rpmlist' elif imageID != None: - + imageID = int(imageID) values['image'] = server.getImageInfo(imageID=imageID) # If/When future image types are supported, add elifs here if needed. if type == 'image': diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index d0e4361d..51f4a78c 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -245,7 +245,7 @@ #end if #if $task.method == 'createLiveCD' and $image - + From bb711506a6e2cdec0601ca654b0dacb8fe492d3c Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Tue, 29 Sep 2009 14:31:32 -0400 Subject: [PATCH 13/24] avoid filename collisions with the user-supplied kickstart file --- builder/kojid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/kojid b/builder/kojid index f9389e54..a61ee01b 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2274,7 +2274,7 @@ class LiveCDTask(BaseTaskHandler): # Write out the new ks file. Note that things may not be in the same # order and comments in the original ks file may be lost. - kskoji = os.path.join(tmpchroot, 'koji.ks') + kskoji = os.path.join(tmpchroot, 'koji-livecd-%s-%i.ks' % (target_info['build_tag_name'], self.id)) kspath = os.path.join(broot.rootdir(), kskoji[1:]) outfile = open(kspath, 'w') outfile.write(str(ks.handler)) From 3adc9c9c40f755bb9df423ee7027b5483ad48959 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Tue, 29 Sep 2009 15:37:23 -0400 Subject: [PATCH 14/24] minor changes, mostly cosmetic --- builder/kojid | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/builder/kojid b/builder/kojid index a61ee01b..7cec9695 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2174,7 +2174,7 @@ class LiveCDTask(BaseTaskHandler): opts = {} global options - target_info = session.getBuildTarget(target) + target_info = session.getBuildTarget(target, strict=True) build_tag = target_info['build_tag'] repo_info = session.getRepo(build_tag) @@ -2289,35 +2289,41 @@ class LiveCDTask(BaseTaskHandler): livecd_log = os.path.join(tmpchroot, 'livecd.log') # path in chroot # Create the loopback devices we need - mknod_cmd = 'for i in `seq 0 7`; do mknod /dev/loop$i b 7 $i; done' + mknod_cmd = 'for i in $(seq 0 7); do mknod /dev/loop$i b 7 $i; done' rv = broot.mock(['--chroot', mknod_cmd]) if rv: broot.expire() raise koji.LiveCDError, \ - "Could not create loopback device files! %S" % rv + "Could not create loopback device files: %s" % _parseStatus(rv, '"%s"' % mknod_cmd) # Run livecd-creator - livecd_cmd = '/usr/bin/livecd-creator -c %s -d -v --logfile=%s --cache=%s' % (kskoji, livecd_log, cachedir) + livecd_cmd = ['/usr/bin/livecd-creator', '-c', kskoji, '-d', '-v', '--logfile', livecd_log, + '--cache', cachedir] # run the livecd-creator command - rv = broot.mock(['--chroot', livecd_cmd]) + rv = broot.mock(['--cwd', '/tmp', '--chroot', '--'] + livecd_cmd) self.uploadFile(os.path.join(broot.rootdir(), livecd_log[1:])) + broot.expire() if rv: - broot.expire() raise koji.LiveCDError, \ - "livecd-creator command failed, see livecd.log or root.log" + "Could not create LiveCD: %s" % _parseStatus(rv, 'livecd-creator') + \ + "; see root.log or livecd.log for more information" # Find the resultant iso - files = os.listdir(broot.rootdir()) + # The cwd of the livecd-creator process is /tmp in the chroot, so that's + # where it writes the .iso + files = os.listdir(os.path.join(broot.rootdir(), 'tmp')) isofile = None for afile in files: - if '.iso' in afile: - isofile = afile - break + if afile.endswith('.iso'): + if not isofile: + isofile = afile + else: + raise koji.LiveCDError, 'multiple .iso files found: %s and %s' % (isofile, afile) if not isofile: raise koji.LiveCDError, 'could not find iso file in chroot' - isosrc = os.path.join(broot.rootdir(), isofile) + isosrc = os.path.join(broot.rootdir(), 'tmp', isofile) try: filesize = os.path.getsize(isosrc) except OSError: @@ -2387,7 +2393,6 @@ class LiveCDTask(BaseTaskHandler): image_id = session.importImage(self.id, isofile, filesize, 'LiveCD ISO', hash, hdrlist) - broot.expire() if opts.get('scratch'): return 'Scratch image created: %s' % \ os.path.join(koji.pathinfo.work(), From b5172f49aef8eab2ad626fbcb41691b8844cd1d6 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Tue, 29 Sep 2009 16:09:08 -0400 Subject: [PATCH 15/24] importImage() should be in HostExports --- builder/kojid | 7 +++---- hub/kojihub.py | 25 ++++++++++++------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/builder/kojid b/builder/kojid index 7cec9695..b48d2784 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2333,8 +2333,7 @@ class LiveCDTask(BaseTaskHandler): # if filesize is greater than a 32-bit signed integer's range, the # python XMLRPC module will break. - if filesize > 2147483647: - filesize = str(filesize) + filesize = str(filesize) # copy the iso out of the chroot. If we were given an isoname, this is # where the renaming happens. @@ -2390,8 +2389,8 @@ class LiveCDTask(BaseTaskHandler): # Import info about the image into the database, unless this is a # scratch image. broot.markExternalRPMs(hdrlist) - image_id = session.importImage(self.id, isofile, filesize, - 'LiveCD ISO', hash, hdrlist) + image_id = session.host.importImage(self.id, isofile, filesize, + 'LiveCD ISO', hash, hdrlist) if opts.get('scratch'): return 'Scratch image created: %s' % \ diff --git a/hub/kojihub.py b/hub/kojihub.py index 5e02466f..bfbd339a 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4598,7 +4598,7 @@ def importImageInternal(task_id, filename, filesize, mediatype, hash, rpmlist): _dml(q, imageinfo) q = """INSERT INTO imageinfo_listing (image_id,rpm_id) - VALUES (%(id)i,%(rpminfo)i)""" + VALUES (%(image_id)i,%(rpm_id)i)""" rpm_ids = [] for an_rpm in rpmlist: @@ -4607,14 +4607,13 @@ def importImageInternal(task_id, filename, filesize, mediatype, hash, rpmlist): data = add_external_rpm(an_rpm, location, strict=False) else: data = get_rpm(an_rpm, strict=True) - rpm_id = data['id'] - rpm_ids.append(rpm_id) + rpm_ids.append(data['id']) + image_id = imageinfo['id'] for rpm_id in rpm_ids: - imageinfo['rpminfo'] = rpm_id - _dml(q, imageinfo) + _dml(q, locals()) - return imageinfo['id'] + return image_id def moveImageResults(task_id, image_id): """ @@ -4755,13 +4754,6 @@ class RootExports(object): ret['filesize'] = str(ret['filesize']) return ret - # Called from kojid::LiveCDTask - def importImage(self, task_id, filename, filesize, mediatype, hash, rpmlist): - image_id = importImageInternal(task_id, filename, filesize, mediatype, - hash, rpmlist) - moveImageResults(task_id, image_id) - return image_id - def hello(self,*args): return "Hello World" @@ -7322,6 +7314,13 @@ class HostExports(object): _untag_build(fromtag,build,user_id=user_id,force=force,strict=True) _tag_build(tag,build,user_id=user_id,force=force) + # Called from kojid::LiveCDTask + def importImage(self, task_id, filename, filesize, mediatype, hash, rpmlist): + image_id = importImageInternal(task_id, filename, filesize, mediatype, + hash, rpmlist) + moveImageResults(task_id, image_id) + return image_id + def tagNotification(self, is_successful, tag_id, from_id, build_id, user_id, ignore_success=False, failure_msg=''): """Create a tag notification message. Handles creation of tagNotification tasks for hosts.""" From bdac9d5f8525ff8bafd2056037ad75bac75e76f6 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Tue, 29 Sep 2009 17:11:53 -0400 Subject: [PATCH 16/24] - store the arch of the image in the imageinfo table - put the non-.iso files in a data/logs/ subdirectory for consistency with builds --- builder/kojid | 2 +- docs/schema.sql | 7 +++-- hub/kojihub.py | 56 ++++++++++++++++++++++--------------- www/kojiweb/imageinfo.chtml | 13 +++------ www/kojiweb/index.py | 15 ++-------- 5 files changed, 47 insertions(+), 46 deletions(-) diff --git a/builder/kojid b/builder/kojid index b48d2784..d153da81 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2389,7 +2389,7 @@ class LiveCDTask(BaseTaskHandler): # Import info about the image into the database, unless this is a # scratch image. broot.markExternalRPMs(hdrlist) - image_id = session.host.importImage(self.id, isofile, filesize, + image_id = session.host.importImage(self.id, isofile, filesize, arch, 'LiveCD ISO', hash, hdrlist) if opts.get('scratch'): diff --git a/docs/schema.sql b/docs/schema.sql index d9e1932b..a6d37d57 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -431,9 +431,11 @@ CREATE TABLE imageinfo ( task_id INTEGER NOT NULL REFERENCES task(id), filename TEXT NOT NULL, filesize BIGINT NOT NULL, + arch VARCHAR(16) NOT NULL, hash TEXT NOT NULL, mediatype VARCHAR(16) NOT NULL ) WITHOUT OIDS; +CREATE INDEX imageinfo_task_id on imageinfo(task_id); -- this table associates tags with builds. an entry here tags a package CREATE TABLE tag_listing ( @@ -587,10 +589,11 @@ CREATE INDEX buildroot_listing_rpms ON buildroot_listing(rpm_id); -- tracks the contents of an image CREATE TABLE imageinfo_listing ( - rpm_id INTEGER NOT NULL REFERENCES rpminfo(id), image_id INTEGER NOT NULL REFERENCES imageinfo(id), - UNIQUE (rpm_id, image_id) + rpm_id INTEGER NOT NULL REFERENCES rpminfo(id), + UNIQUE (image_id, rpm_id) ) WITHOUT OIDS; +CREATE INDEX imageinfo_listing_rpms on imageinfo_listing(rpm_id); CREATE TABLE log_messages ( id SERIAL NOT NULL PRIMARY KEY, diff --git a/hub/kojihub.py b/hub/kojihub.py index bfbd339a..a87317d5 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4569,7 +4569,7 @@ def rpmdiff(basepath, rpmlist): raise koji.BuildError, 'mismatch when analyzing %s, rpmdiff output was:\n%s' % \ (os.path.basename(first_rpm), output) -def importImageInternal(task_id, filename, filesize, mediatype, hash, rpmlist): +def importImageInternal(task_id, filename, filesize, arch, mediatype, hash, rpmlist): """ Import image info and the listing into the database, and move an image to the final resting place. The filesize may be reported as a string if it @@ -4588,12 +4588,13 @@ def importImageInternal(task_id, filename, filesize, mediatype, hash, rpmlist): imageinfo['taskid'] = task_id imageinfo['filename'] = filename imageinfo['filesize'] = int(filesize) + imageinfo['arch'] = arch imageinfo['mediatype'] = mediatype imageinfo['hash'] = hash q = """INSERT INTO imageinfo (id,task_id,filename,filesize, - mediatype,hash) + arch,mediatype,hash) VALUES (%(id)i,%(taskid)i,%(filename)s,%(filesize)i, - %(mediatype)s,%(hash)s) + %(arch)s,%(mediatype)s,%(hash)s) """ _dml(q, imageinfo) @@ -4615,7 +4616,7 @@ def importImageInternal(task_id, filename, filesize, mediatype, hash, rpmlist): return image_id -def moveImageResults(task_id, image_id): +def moveImageResults(task_id, image_id, arch): """ Move the image file from the work/task directory into its more permanent resting place. This shouldn't be called for scratch images. @@ -4624,23 +4625,28 @@ def moveImageResults(task_id, image_id): koji.pathinfo.taskrelpath(task_id)) final_path = os.path.join(koji.pathinfo.imageFinalPath(), koji.pathinfo.livecdRelPath(image_id)) - src_files = os.listdir(source_path) - if os.path.exists(final_path): - raise koji.GenericError("Error moving LiveCD image: the final " + - "destination already exists!") + log_path = os.path.join(final_path, 'data', 'logs', arch) + if os.path.exists(final_path) or os.path.exists(log_path): + raise koji.GenericError, "Error moving LiveCD image: the final " + \ + "destination already exists!" koji.ensuredir(final_path) + koji.ensuredir(log_path) + src_files = os.listdir(source_path) got_iso = False for fname in src_files: - if '.iso' in fname: got_iso = True + if fname.endswith('.iso'): + got_iso = True + dest_path = final_path + else: + dest_path = log_path os.rename(os.path.join(source_path, fname), - os.path.join(final_path, fname)) - os.symlink(os.path.join(final_path, fname), + os.path.join(dest_path, fname)) + os.symlink(os.path.join(dest_path, fname), os.path.join(source_path, fname)) if not got_iso: - raise koji.GenericError( - "Could not move the iso to the final destination!") + raise koji.GenericError, "Could not move the iso to the final destination!" # # XMLRPC Methods @@ -4723,7 +4729,7 @@ class RootExports(object): # Database access to get imageinfo values. Used in parts of kojiweb. # - def getImageInfo(self, imageID=None, taskID=None): + def getImageInfo(self, imageID=None, taskID=None, strict=False): """ Return the row from imageinfo given an image_id OR build_root_id. It is an error if neither are specified, and image_id takes precedence. @@ -4731,24 +4737,30 @@ class RootExports(object): integer limit. """ tables = ['imageinfo'] - fields = ['imageinfo.id', 'filename', 'filesize', 'mediatype', + fields = ['imageinfo.id', 'filename', 'filesize', 'imageinfo.arch', 'mediatype', 'imageinfo.task_id', 'buildroot.id', 'hash'] - aliases = ['id', 'filename', 'filesize', 'mediatype', 'task_id', + aliases = ['id', 'filename', 'filesize', 'arch', 'mediatype', 'task_id', 'br_id', 'hash'] joins = ['buildroot ON imageinfo.task_id = buildroot.task_id'] if imageID: clauses = ['imageinfo.id = %(imageID)i'] elif taskID: clauses = ['imageinfo.task_id = %(taskID)i'] + else: + raise koji.GenericError, 'either imageID or taskID must be specified' query = QueryProcessor(columns=fields, tables=tables, clauses=clauses, values=locals(), joins=joins, aliases=aliases) ret = query.executeOne() + if strict and not ret: + if imageID: + raise koji.GenericError, 'no image with ID: %i' % imageID + else: + raise koji.GenericError, 'no image for task ID: %i' % taskID + # additional tweaking if ret: - ret['path'] = os.path.join(koji.pathinfo.imageFinalPath(), - koji.pathinfo.livecdRelPath(ret['id'])) # Always return filesize as a string instead of an int so XMLRPC doesn't # complain about 32-bit overflow ret['filesize'] = str(ret['filesize']) @@ -5675,7 +5687,7 @@ class RootExports(object): # image specific constraints if imageID != None: - clauses.append('imageinfo_listing.image_id = %(imageID)s') + clauses.append('imageinfo_listing.image_id = %(imageID)i') joins.append('imageinfo_listing ON rpminfo.id = imageinfo_listing.rpm_id') if hostID != None: @@ -7315,10 +7327,10 @@ class HostExports(object): _tag_build(tag,build,user_id=user_id,force=force) # Called from kojid::LiveCDTask - def importImage(self, task_id, filename, filesize, mediatype, hash, rpmlist): - image_id = importImageInternal(task_id, filename, filesize, mediatype, + def importImage(self, task_id, filename, filesize, arch, mediatype, hash, rpmlist): + image_id = importImageInternal(task_id, filename, filesize, arch, mediatype, hash, rpmlist) - moveImageResults(task_id, image_id) + moveImageResults(task_id, image_id, arch) return image_id def tagNotification(self, is_successful, tag_id, from_id, build_id, user_id, ignore_success=False, failure_msg=''): diff --git a/www/kojiweb/imageinfo.chtml b/www/kojiweb/imageinfo.chtml index f8ebd26b..d85bd479 100644 --- a/www/kojiweb/imageinfo.chtml +++ b/www/kojiweb/imageinfo.chtml @@ -17,6 +17,9 @@ + + + @@ -41,19 +44,11 @@ - - - - - +
Image ID$image.id
Build Root ID$image.br_idID$image.id
File Name$image.filename
File Size$image.filesize bytes
File Size$image.filesize
Media Type$image.mediatype
Hash (MD5)$image.hashDigest (md5)$image.hashHash (SHA1)$image.hashDigest (sha1)$image.hashHash (SHA256)$image.hashDigest (sha256)$image.hashHash (SHA384)$image.hashDigest (sha384)$image.hashHash (SHA512)$image.hashDigest (sha512)$image.hashHash $image.hash
Task ID$image.task_idTask$koji.taskLabel($task)
Buildroot/var/lib/mock/$buildroot.tag_name-$buildroot.id-$buildroot.repo_id
Output
#for $ofile in $logs -     $basename($ofile)
+ $basename($ofile)
#end for
Image InformationImage $image.filename
File Size$image.filesize
Arch$image.arch
Media Type$image.mediatype
Buildroot/var/lib/mock/$buildroot.tag_name-$buildroot.id-$buildroot.repo_id
Output - #for $ofile in $logs - $basename($ofile)
- #end for -
Included RPMs
Download ImageDownload Image (build logs)
diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 9a1871b2..3666c736 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -573,21 +573,12 @@ def imageinfo(req, imageID): values = _initValues(req, 'Image Information') imageURL = req.get_options().get('KojiImagesURL', 'http://localhost/images') imageID = int(imageID) - image = server.getImageInfo(imageID=imageID) + image = server.getImageInfo(imageID=imageID, strict=True) values['image'] = image values['title'] = image['filename'] + ' | Image Information' - urlrelpath = koji.pathinfo.livecdRelPath(image['id']) - values['buildroot'] = server.getBuildroot(image['br_id']) + values['buildroot'] = server.getBuildroot(image['br_id'], strict=True) values['task'] = server.getTaskInfo(image['task_id'], request=True) - filelist = [] - for ofile in os.listdir(values['image']['path']): - relpath = os.path.join(urlrelpath, ofile) - if relpath.endswith('.iso'): - values['imageURL'] = imageURL + '/' + relpath - else: - filelist.append(imageURL + '/' + relpath) - - values['logs'] = filelist + values['imageBase'] = imageURL + '/' + koji.pathinfo.livecdRelPath(image['id']) return _genHTML(req, 'imageinfo.chtml') def taskstatus(req, taskID): From cfa34bc744adb8a0ea26e63543963a5638a6a58b Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Tue, 29 Sep 2009 18:02:45 -0400 Subject: [PATCH 17/24] rename the spin-livecd option --scmurl to --ksurl (in case we allow specifying a --compsurl later) --- builder/kojid | 6 +++--- cli/koji | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/builder/kojid b/builder/kojid index d153da81..11fae0f5 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2212,13 +2212,13 @@ class LiveCDTask(BaseTaskHandler): # path to it in the /mnt/koji/work directory. If not, then it is still # the parameter the user passed in initially, and we assume it is a # relative path in a remote scm. The user should have passed in an scm - # url with --scmurl. + # url with --ksurl. tmpchroot = '/tmp' scmdir = os.path.join(broot.rootdir(), tmpchroot[1:]) koji.ensuredir(scmdir) self.logger.debug("ksfile = %s" % ksfile) - if opts.get('scmurl'): - scm = SCM(opts['scmurl']) + if opts.get('ksurl'): + scm = SCM(opts['ksurl']) scm.assert_allowed(options.allowed_scms) logfile = os.path.join(self.workdir, 'checkout.log') scmsrcdir = scm.checkout(scmdir, self.getUploadDir(), logfile) diff --git a/cli/koji b/cli/koji index 4f5a150d..f99dbb57 100755 --- a/cli/koji +++ b/cli/koji @@ -3830,8 +3830,8 @@ def handle_spin_livecd(options, session, args): help=_("Run the livecd creation task at a lower priority")) parser.add_option("--isoname", help=_("Use a custom name for the iso file")) - parser.add_option("--scmurl", - help=_("The SCM URL to the kickstart file")) + parser.add_option("--ksurl", + help=_("The URL to the SCM containing the kickstart file")) parser.add_option("--scratch", action="store_true", help=_("Create a scratch LiveCD image.")) parser.add_option("--repo", @@ -3878,14 +3878,14 @@ def handle_spin_livecd(options, session, args): # If it's a URL, it's kojid's job to go get it when it does the checkout. ksfile = args[2] - if not task_options.scmurl: + if not task_options.ksurl: serverdir = _unique_path('cli-livecd') session.uploadWrapper(ksfile, serverdir, callback=callback) ksfile = os.path.join(serverdir, os.path.basename(ksfile)) print livecd_opts = {} - for opt in ['scratch', 'scmurl', 'isoname', 'repo']: + for opt in ['scratch', 'ksurl', 'isoname', 'repo']: if getattr(task_options, opt): livecd_opts[opt] = getattr(task_options, opt) From dd0f48e752779d6a28b2e845e00f1545bae03462 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Tue, 29 Sep 2009 18:05:34 -0400 Subject: [PATCH 18/24] don't depend on a specific version of a pykickstart RepoData class --- builder/kojid | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/builder/kojid b/builder/kojid index 11fae0f5..6414d4f9 100755 --- a/builder/kojid +++ b/builder/kojid @@ -59,8 +59,7 @@ from optparse import OptionParser from StringIO import StringIO from xmlrpclib import Fault import pykickstart.parser as ksparser -import pykickstart.version as ksversion -import pykickstart.commands.repo as ksrepo +import pykickstart.handlers.control as kscontrol import hashlib import iso9660 # from pycdio @@ -2241,8 +2240,10 @@ class LiveCDTask(BaseTaskHandler): # in the kickstart file. If --repo wasn't specified, then we use the # repo associated with the target passed in initially. self.uploadFile(kspath) # upload the original ks file - version = ksversion.makeVersion() + version = ksparser.makeVersion() ks = ksparser.KickstartParser(version) + repo_class = kscontrol.dataMap[ks.version]['RepoData'] + try: ks.readKickstart(kspath) except IOError, (err, msg): @@ -2257,8 +2258,8 @@ class LiveCDTask(BaseTaskHandler): user_repos = opts['repo'].split(',') index = 0 for user_repo in user_repos: - ks.handler.repo.repoList.append(ksrepo.F8_RepoData( - baseurl=user_repo, name='koji_override_%s' % index)) + ks.handler.repo.repoList.append(repo_class(baseurl=user_repo, + name='koji-override-%i' % index)) index += 1 else: topurl = getattr(options, 'topurl') @@ -2269,8 +2270,9 @@ class LiveCDTask(BaseTaskHandler): target_info['build_tag_name']) baseurl = '%s/%s' % (repopath, arch) self.logger.debug('BASEURL: %s' % baseurl) - ks.handler.repo.repoList.append(ksrepo.F8_RepoData( - baseurl=baseurl, name='koji')) + ks.handler.repo.repoList.append(repo_class(baseurl=baseurl, + name='koji-%s-%i' % (target_info['build_tag_name'], + repo_info['id']))) # Write out the new ks file. Note that things may not be in the same # order and comments in the original ks file may be lost. From 82e38da8e9802413c88c7529c51e3d2a21cec525 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Tue, 29 Sep 2009 18:29:12 -0400 Subject: [PATCH 19/24] enable specifying the syntax version of the kickstart file --- builder/kojid | 5 ++++- cli/koji | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/builder/kojid b/builder/kojid index 6414d4f9..dd316a0e 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2240,7 +2240,10 @@ class LiveCDTask(BaseTaskHandler): # in the kickstart file. If --repo wasn't specified, then we use the # repo associated with the target passed in initially. self.uploadFile(kspath) # upload the original ks file - version = ksparser.makeVersion() + if opts.get('ksversion'): + version = ksparser.makeVersion(ksparser.stringToVersion(opts['ksversion'])) + else: + version = ksparser.makeVersion() ks = ksparser.KickstartParser(version) repo_class = kscontrol.dataMap[ks.version]['RepoData'] diff --git a/cli/koji b/cli/koji index f99dbb57..943b4926 100755 --- a/cli/koji +++ b/cli/koji @@ -3830,8 +3830,10 @@ def handle_spin_livecd(options, session, args): help=_("Run the livecd creation task at a lower priority")) parser.add_option("--isoname", help=_("Use a custom name for the iso file")) - parser.add_option("--ksurl", + parser.add_option("--ksurl", metavar="SCMURL", help=_("The URL to the SCM containing the kickstart file")) + parser.add_option("--ksversion", metavar="VERSION", + help=_("The syntax version used in the kickstart file")) parser.add_option("--scratch", action="store_true", help=_("Create a scratch LiveCD image.")) parser.add_option("--repo", @@ -3885,7 +3887,7 @@ def handle_spin_livecd(options, session, args): print livecd_opts = {} - for opt in ['scratch', 'ksurl', 'isoname', 'repo']: + for opt in ['scratch', 'ksurl', 'ksversion', 'isoname', 'repo']: if getattr(task_options, opt): livecd_opts[opt] = getattr(task_options, opt) From 0a9142ff6c183a8cda047a27a32b06234fd3ca01 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Wed, 30 Sep 2009 09:31:04 -0400 Subject: [PATCH 20/24] allow longer mediatypes --- docs/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/schema.sql b/docs/schema.sql index a6d37d57..9da1afd5 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -433,7 +433,7 @@ CREATE TABLE imageinfo ( filesize BIGINT NOT NULL, arch VARCHAR(16) NOT NULL, hash TEXT NOT NULL, - mediatype VARCHAR(16) NOT NULL + mediatype TEXT NOT NULL ) WITHOUT OIDS; CREATE INDEX imageinfo_task_id on imageinfo(task_id); From 1e92e0d510cc472b7fb59067b95b8d86894268c6 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Wed, 30 Sep 2009 10:32:51 -0400 Subject: [PATCH 21/24] add new Requires to koji-builder --- koji.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/koji.spec b/koji.spec index 28db7c3f..8fb2f8e8 100644 --- a/koji.spec +++ b/koji.spec @@ -58,6 +58,8 @@ Requires: /usr/bin/svn Requires: /usr/bin/git Requires: rpm-build Requires: redhat-rpm-config +Requires: pykickstart +Requires: pycdio %if 0%{?rhel} >= 5 Requires: createrepo >= 0.4.11-2 %endif From a66e628f10623569cb8559893250161a08264fdf Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Wed, 30 Sep 2009 15:40:21 -0400 Subject: [PATCH 22/24] - include directory entries in the manifest - don't call iso9660.name_translate(), it seems unnecessary --- builder/kojid | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/builder/kojid b/builder/kojid index dd316a0e..8e2ea802 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1500,18 +1500,27 @@ class BaseTaskHandler(object): manifest = [] file_stats = iso.readdir(path) for stat in file_stats: - filename = stat[0] size = stat[2] - is_dir = stat[4] - if filename == "." or filename == "..": continue + is_dir = stat[4] == 2 - if is_dir == 1: - manifest.append("%-10d %s\n" % (size, os.path.join(path, - iso9660.name_translate(filename)))) + if filename == '..': + continue + elif filename == '.': + # path should always end in a trailing / + filepath = path else: - manifest.extend(self.listImageDir(iso, - os.path.join(path, filename))) + filepath = path + filename + # identify directories with a trailing / + if is_dir: + filepath += '/' + + if is_dir and filename != '.': + # recurse into subdirectories + manifest.extend(self.listImageDir(iso, filepath)) + else: + # output information for the current directory and files + manifest.append("%-10d %s\n" % (size, filepath)) return manifest From 7e999d1480d371bff905a237f5bcc1703eb38131 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Wed, 30 Sep 2009 17:04:56 -0400 Subject: [PATCH 23/24] handle all strings as unicode for compatibility with Cheetah 2.2.x (also backward-compatible with 2.0.x) --- www/kojiweb/includes/header.chtml | 39 ++++++++++++++++--------------- www/lib/kojiweb/util.py | 14 +++++++++-- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/www/kojiweb/includes/header.chtml b/www/kojiweb/includes/header.chtml index 7be84f90..3f7efca3 100644 --- a/www/kojiweb/includes/header.chtml +++ b/www/kojiweb/includes/header.chtml @@ -1,3 +1,4 @@ +#encoding utf-8 #import koji #import random Date: Wed, 30 Sep 2009 17:05:23 -0400 Subject: [PATCH 24/24] avoid double-escaping ampersands --- www/lib/kojiweb/util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index 47e46fa0..3e24e3b4 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -46,6 +46,7 @@ class XHTMLFilter(DecodeUTF8): def filter(self, *args, **kw): result = super(XHTMLFilter, self).filter(*args, **kw) result = result.replace('&', '&') + result = result.replace('&amp;', '&') result = result.replace('&nbsp;', ' ') result = result.replace('&lt;', '<') result = result.replace('&gt;', '>')