From 0f727a2ab4a4d0c568fee0b74ab9771ea8c5a392 Mon Sep 17 00:00:00 2001 From: Yuming Zhu Date: Wed, 26 Feb 2020 02:31:31 +0800 Subject: [PATCH] flake8: apply E3 rules --- .flake8 | 2 +- builder/kojid | 40 +++---- builder/mergerepos | 3 + cli/koji_cli/commands.py | 6 +- cli/koji_cli/lib.py | 2 +- hub/kojihub.py | 205 +++++++++++++++++++++++++++------- hub/kojixmlrpc.py | 7 ++ koji/__init__.py | 78 +++++++++++++ koji/arch.py | 21 ++++ koji/auth.py | 11 +- koji/context.py | 1 + koji/daemon.py | 3 + koji/db.py | 4 + koji/plugin.py | 6 + koji/policy.py | 4 + koji/rpmdiff.py | 1 + koji/server.py | 2 + koji/tasks.py | 18 ++- koji/util.py | 3 +- koji/xmlrpcplus.py | 1 - plugins/hub/protonmsg.py | 12 ++ plugins/hub/runroot_hub.py | 1 + setup.py | 1 + util/koji-gc | 21 ++++ util/koji-shadow | 12 ++ util/koji-sweep-db | 1 - util/kojira | 6 +- vm/kojikamid.py | 14 +++ vm/kojivmd | 8 ++ www/kojiweb/index.py | 70 ++++++++++++ www/kojiweb/wsgi_publisher.py | 1 - www/lib/kojiweb/util.py | 39 +++++++ 32 files changed, 534 insertions(+), 70 deletions(-) diff --git a/.flake8 b/.flake8 index 3c692926..0fc72e64 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -select = I,C,F,E1,E2 +select = I,C,F,E1,E2,E3 ignore = E266 exclude = .git, diff --git a/builder/kojid b/builder/kojid index 7ce610d6..ca7459b0 100755 --- a/builder/kojid +++ b/builder/kojid @@ -127,6 +127,7 @@ try: except ImportError: # pragma: no cover ozif_enabled = False + def main(options, session): logger = logging.getLogger("koji.build") logger.info('Starting up') @@ -140,8 +141,10 @@ def main(options, session): for name in options.plugin: logger.info('Loading plugin: %s' % name) tm.scanPlugin(pt.load(name)) + def shutdown(*args): raise SystemExit + def restart(*args): logger.warn("Initiating graceful restart") tm.restart_pending = True @@ -590,7 +593,6 @@ class BuildRoot(object): self.expire() raise koji.BuildError("error building srpm, %s" % self._mockResult(rv)) - def build_srpm(self, specfile, sourcedir, source_cmd): self.session.host.setBuildRootState(self.id, 'BUILDING') if source_cmd: @@ -1162,7 +1164,6 @@ class BuildTask(BaseTaskHandler): raise koji.BuildError("No matching arches were found") return to_list(archdict.keys()) - def choose_taskarch(self, arch, srpm, build_tag): """Adjust the arch for buildArch subtask as needed""" if koji.util.multi_fnmatch(arch, self.options.literal_task_arches): @@ -1203,7 +1204,6 @@ class BuildTask(BaseTaskHandler): # otherwise, noarch is ok return 'noarch' - def runBuilds(self, srpm, build_tag, archlist, repo_id, failany=True): self.logger.debug("Spawning jobs for arches: %r" % (archlist)) subtasks = {} @@ -1456,6 +1456,7 @@ class BuildArchTask(BaseBuildTask): return ret + class MavenTask(MultiPlatformTask): Methods = ['maven'] @@ -1542,6 +1543,7 @@ class MavenTask(MultiPlatformTask): arch='noarch') self.wait(tag_task_id) + class BuildMavenTask(BaseBuildTask): Methods = ['buildMaven'] @@ -1633,7 +1635,6 @@ class BuildMavenTask(BaseBuildTask): logfile = self.workdir + '/checkout.log' uploadpath = self.getUploadDir() - self.run_callbacks('preSCMCheckout', scminfo=scm.get_info(), build_tag=build_tag, scratch=opts.get('scratch')) # Check out sources from the SCM sourcedir = scm.checkout(scmdir, self.session, uploadpath, logfile) @@ -1767,6 +1768,7 @@ class BuildMavenTask(BaseBuildTask): 'logs': logs, 'files': output_files} + class WrapperRPMTask(BaseBuildTask): """Build a wrapper rpm around archives output from a Maven or Windows build. May either be called as a subtask or as a separate @@ -2118,6 +2120,7 @@ class WrapperRPMTask(BaseBuildTask): return results + class ChainMavenTask(MultiPlatformTask): Methods = ['chainmaven'] @@ -2328,6 +2331,7 @@ class ChainMavenTask(MultiPlatformTask): # everything matches return build + class TagBuildTask(BaseTaskHandler): Methods = ['tagBuild'] @@ -2352,6 +2356,7 @@ class TagBuildTask(BaseTaskHandler): self.session.host.tagNotification(False, tag_id, fromtag, build_id, user_id, ignore_success, "%s: %s" % (exctype, value)) raise e + class BuildImageTask(MultiPlatformTask): def initImageBuild(self, name, version, release, target_info, opts): @@ -2372,6 +2377,7 @@ class BuildImageTask(MultiPlatformTask): """return the next available release number for an N-V""" return self.session.getNextRelease(dict(name=name, version=ver)) + class BuildBaseImageTask(BuildImageTask): Methods = ['image'] @@ -2524,7 +2530,6 @@ class BuildApplianceTask(BuildImageTask): if koji.canonArch(arch) not in tag_archlist: raise koji.BuildError("Invalid arch for build tag: %s" % arch) - if not opts: opts = {} @@ -2593,6 +2598,7 @@ class BuildApplianceTask(BuildImageTask): report += 'appliance build results in: %s' % respath return report + class BuildLiveCDTask(BuildImageTask): Methods = ['livecd'] @@ -2727,7 +2733,6 @@ class BuildLiveMediaTask(BuildImageTask): canfail.append(subtasks[arch]) self.logger.debug("Tasks that can fail: %r", canfail) - self.logger.debug("Got image subtasks: %r", subtasks) self.logger.debug("Waiting on livemedia subtasks...") results = self.wait(to_list(subtasks.values()), all=True, failany=True, canfail=canfail) @@ -3136,6 +3141,8 @@ class ApplianceTask(ImageTask): # via the livecd-build group. livecd-creator is then executed in the chroot # to create the LiveCD image. # + + class LiveCDTask(ImageTask): Methods = ['createLiveCD'] @@ -3208,7 +3215,6 @@ class LiveCDTask(ImageTask): return manifest - def handler(self, name, version, release, arch, target_info, build_tag, repo_info, ksfile, opts=None): if opts == None: @@ -3288,7 +3294,6 @@ class LiveCDTask(ImageTask): return imgdata - # livemedia-creator class LiveMediaTask(ImageTask): @@ -3406,7 +3411,6 @@ class LiveMediaTask(ImageTask): livemedia_log = broot.tmpdir(within=True) + '/lmc-logs/livemedia-out.log' resultdir = broot.tmpdir(within=True) + '/lmc' - # Common LMC command setup, needs extending cmd = ['/sbin/livemedia-creator', '--ks', kskoji, @@ -3417,7 +3421,6 @@ class LiveMediaTask(ImageTask): # '--tmp', '/tmp' ] - volid = opts.get('volid') if not volid: volid = self._shortenVolID(name, version, release) @@ -3437,7 +3440,6 @@ class LiveMediaTask(ImageTask): '--releasever', version, ]) - if arch == 'x86_64': cmd.append('--macboot') @@ -3445,7 +3447,6 @@ class LiveMediaTask(ImageTask): templates_dir = self.fetch_lorax_templates_from_scm(broot) cmd.extend(['--lorax-templates', templates_dir]) - # Run livemedia-creator rv = broot.mock(['--cwd', broot.tmpdir(within=True), '--chroot', '--'] + cmd) @@ -3490,7 +3491,6 @@ class LiveMediaTask(ImageTask): raise koji.LiveMediaError('could not find iso file in chroot') isosrc = os.path.join(rootresultsdir, isofile) - # Generate the file manifest of the image, upload the results manifest = os.path.join(broot.resultdir(), 'manifest.log') self.genISOManifest(isosrc, manifest) @@ -3522,6 +3522,8 @@ class LiveMediaTask(ImageTask): # A generic task for building disk images using Oz # Other Oz-based image handlers should inherit this. + + class OzImageTask(BaseTaskHandler): Methods = [] @@ -3757,7 +3759,6 @@ class OzImageTask(BaseTaskHandler): raise koji.BuildError('Unknown or supported distro given: %s' % distro) def fixImageXML(self, format, filename, xmltext): - """ The XML generated by Oz/ImageFactory knows nothing about the name or image format conversions Koji does. We fix those values in the @@ -3803,6 +3804,7 @@ class OzImageTask(BaseTaskHandler): screenshot = found[0] return screenshot + class BaseImageTask(OzImageTask): Methods = ['createImage'] @@ -4003,7 +4005,6 @@ class BaseImageTask(OzImageTask): logerror=1) return {'image': newimg} - def _buildTarGZ(self, format): """ Use tar and gzip to compress a raw disk image. @@ -4034,7 +4035,6 @@ class BaseImageTask(OzImageTask): return {'image': newimg} - def _buildSquashfs(self, format): """ Use squashfs to wrap a raw disk image into liveimg compatible image. @@ -4309,6 +4309,7 @@ class BaseImageTask(OzImageTask): # no need to delete anything since self.workdir will get scrubbed return imgdata + class BuildIndirectionImageTask(OzImageTask): Methods = ['indirectionimage'] @@ -4550,7 +4551,6 @@ class BuildIndirectionImageTask(OzImageTask): # reraise the exception raise - def _do_indirection(self, opts, base_factory_image, utility_factory_image, indirection_template, tlog, ozlog, fhandler, bld_info, target_info, bd): @@ -4861,6 +4861,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask): 'source': source, } + class TagNotificationTask(BaseTaskHandler): Methods = ['tagNotification'] @@ -4952,6 +4953,7 @@ Status: %(status)s\r return 'sent notification of tag operation %i to: %s' % (self.id, to_addrs) + class BuildNotificationTask(BaseTaskHandler): Methods = ['buildNotification'] @@ -5633,7 +5635,6 @@ class createDistRepoTask(BaseTaskHandler): raise koji.GenericError('failed to create repo: %s' \ % parseStatus(status, ' '.join(cmd))) - def do_multilib_dnf(self, arch, ml_arch, conf): repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) mldir = os.path.join(repodir, koji.canonArch(ml_arch)) @@ -5760,7 +5761,6 @@ enabled=1 rpminfo['_multilib'] = True self.kojipkgs[bnp] = rpminfo - def do_multilib_yum(self, arch, ml_arch, conf): repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name']) mldir = os.path.join(repodir, koji.canonArch(ml_arch)) @@ -6303,6 +6303,7 @@ def get_options(): return options + def quit(msg=None, code=1): if msg: logging.getLogger("koji.build").error(msg) @@ -6310,6 +6311,7 @@ def quit(msg=None, code=1): sys.stderr.flush() sys.exit(code) + if __name__ == "__main__": koji.add_file_logger("koji", "/var/log/kojid.log") # note we're setting logging params for all of koji* diff --git a/builder/mergerepos b/builder/mergerepos index 0e59ed08..3b806d51 100755 --- a/builder/mergerepos +++ b/builder/mergerepos @@ -57,6 +57,7 @@ MULTILIB_ARCHES = { 's390x': 's390' } + def parse_args(args): """Parse our opts/args""" usage = """ @@ -337,6 +338,7 @@ class RepoMerge(object): mdgen.doRepoMetadata() mdgen.doFinalMove() + def main(args): """main""" opts = parse_args(args) @@ -358,5 +360,6 @@ def main(args): finally: merge.close() + if __name__ == "__main__": main(sys.argv[1:]) diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index c1a72ff9..62775679 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -2617,6 +2617,7 @@ def anon_handle_list_groups(goptions, session, args): groups = [x[1] for x in tmp_list] tags_cache = {} + def get_cached_tag(tag_id): if tag_id not in tags_cache: tag = session.getTag(tag_id, strict=False) @@ -4311,6 +4312,7 @@ def _print_histline(entry, **kwargs): dkey = key print(" %s: %s" % (dkey, x[key])) + _table_keys = { 'user_perms': ['user_id', 'perm_id'], 'user_groups': ['user_id', 'group_id'], @@ -5635,7 +5637,6 @@ def handle_image_build_indirection(options, session, args): parser.add_option("--noprogress", action="store_true", help=_("Do not display progress of the upload")) - (task_options, args) = parser.parse_args(args) _build_image_indirection(options, task_options, session, args) @@ -5694,7 +5695,6 @@ def _build_image_indirection(options, task_opts, session, args): # Set the architecture task_opts.arch = koji.canonArch(task_opts.arch) - # Upload the indirection template file to the staging area. # If it's a URL, it's kojid's job to go get it when it does the checkout. if not task_opts.indirection_template_url: @@ -6907,7 +6907,6 @@ def anon_handle_wait_repo(options, session, args): return 1 tag_id = tag_info['id'] - for nvr in builds: data = session.getLatestBuilds(tag_id, package=nvr["name"]) if len(data) == 0: @@ -7141,6 +7140,7 @@ def handle_dist_repo(options, session, args): _search_types = ('package', 'build', 'tag', 'target', 'user', 'host', 'rpm', 'maven', 'win') + def anon_handle_search(options, session, args): "[search] Search the system" usage = _("usage: %prog search [options] ") diff --git a/cli/koji_cli/lib.py b/cli/koji_cli/lib.py index b2b26238..cca44b02 100644 --- a/cli/koji_cli/lib.py +++ b/cli/koji_cli/lib.py @@ -105,6 +105,7 @@ Available categories are: %(categories)s def get_usage_str(usage): return usage + _("\n(Specify the --help global option for a list of other help options)") + def ensure_connection(session): try: ret = session.getAPIVersion() @@ -380,7 +381,6 @@ def watch_logs(session, tasklist, opts, poll_interval): lastlog = currlog bytes_to_stdout(contents) - if opts.follow: for child in session.getTaskChildren(task_id): if child['id'] not in tasklist: diff --git a/hub/kojihub.py b/hub/kojihub.py index af2cc58c..c1ac40ba 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -84,6 +84,7 @@ logger = logging.getLogger('koji.hub') NUMERIC_TYPES = tuple(list(six.integer_types) + [float]) + def log_error(msg): logger.error(msg) @@ -92,6 +93,7 @@ def xform_user_krb(entry): entry['krb_principals'] = [x for x in entry['krb_principals'] if x is not None] return entry + class Task(object): """A task for the build hosts""" @@ -512,6 +514,7 @@ class Task(object): koji.plugin.run_callbacks(cbtype, attribute=attr, old=old_val, new=new_val, info=info) + def make_task(method, arglist, **opts): """Create a task @@ -647,6 +650,7 @@ def make_task(method, arglist, **opts): koji.plugin.run_callbacks('postTaskStateChange', attribute='state', old=None, new='FREE', info=opts) return task_id + def eventCondition(event, table=None): """return the proper WHERE condition to select data at the time specified by event. """ if not table: @@ -661,6 +665,7 @@ def eventCondition(event, table=None): else: raise koji.GenericError("Invalid event: %r" % event) + def readGlobalInheritance(event=None): c = context.cnx.cursor() fields = ('tag_id', 'parent_id', 'name', 'priority', 'maxdepth', 'intransitive', @@ -673,6 +678,7 @@ def readGlobalInheritance(event=None): # convert list of lists into a list of dictionaries return [dict(zip(fields, x)) for x in c.fetchall()] + def readInheritanceData(tag_id, event=None): c = context.cnx.cursor() fields = ('parent_id', 'name', 'priority', 'maxdepth', 'intransitive', 'noconfig', 'pkg_filter') @@ -688,6 +694,7 @@ def readInheritanceData(tag_id, event=None): datum['child_id'] = tag_id return data + def readDescendantsData(tag_id, event=None): c = context.cnx.cursor() fields = ('tag_id', 'parent_id', 'name', 'priority', 'maxdepth', 'intransitive', 'noconfig', 'pkg_filter') @@ -794,6 +801,7 @@ def _writeInheritanceData(tag_id, changes, clear=False): insert.make_create() insert.execute() + def readFullInheritance(tag_id, event=None, reverse=False, stops=None, jumps=None): """Returns a list representing the full, ordered inheritance from tag""" if stops is None: @@ -804,6 +812,7 @@ def readFullInheritance(tag_id, event=None, reverse=False, stops=None, jumps=Non readFullInheritanceRecurse(tag_id, event, order, stops, {}, {}, 0, None, False, [], reverse, jumps) return order + def readFullInheritanceRecurse(tag_id, event, order, prunes, top, hist, currdepth, maxdepth, noconfig, pfilter, reverse, jumps): if maxdepth is not None and maxdepth < 1: return @@ -913,12 +922,14 @@ def _pkglist_remove(tag_id, pkg_id): update.make_revoke() # XXX user_id? update.execute() + def _pkglist_owner_remove(tag_id, pkg_id): clauses = ('package_id=%(pkg_id)i', 'tag_id=%(tag_id)i') update = UpdateProcessor('tag_package_owners', values=locals(), clauses=clauses) update.make_revoke() # XXX user_id? update.execute() + def _pkglist_owner_add(tag_id, pkg_id, owner): _pkglist_owner_remove(tag_id, pkg_id) data = {'tag_id': tag_id, 'package_id': pkg_id, 'owner': owner} @@ -926,6 +937,7 @@ def _pkglist_owner_add(tag_id, pkg_id, owner): insert.make_create() # XXX user_id? insert.execute() + def _pkglist_add(tag_id, pkg_id, owner, block, extra_arches): # revoke old entry (if present) _pkglist_remove(tag_id, pkg_id) @@ -940,6 +952,7 @@ def _pkglist_add(tag_id, pkg_id, owner, block, extra_arches): insert.execute() _pkglist_owner_add(tag_id, pkg_id, owner) + def pkglist_add(taginfo, pkginfo, owner=None, block=None, extra_arches=None, force=False, update=False): """Add to (or update) package list for tag""" return _direct_pkglist_add(taginfo, pkginfo, owner, block, extra_arches, @@ -1028,6 +1041,7 @@ def _direct_pkglist_add(taginfo, pkginfo, owner, block, extra_arches, force, block=block, extra_arches=extra_arches, force=force, update=update, user=user) + def pkglist_remove(taginfo, pkginfo, force=False): """Remove package from the list for tag @@ -1065,6 +1079,7 @@ def pkglist_block(taginfo, pkginfo, force=False): (pkg['name'], tag['name'])) pkglist_add(taginfo, pkginfo, block=True, force=force) + def pkglist_unblock(taginfo, pkginfo, force=False): """Unblock the package in tag @@ -1101,14 +1116,17 @@ def pkglist_unblock(taginfo, pkginfo, force=False): _pkglist_add(tag_id, pkg_id, previous['owner_id'], False, previous['extra_arches']) koji.plugin.run_callbacks('postPackageListChange', action='unblock', tag=tag, package=pkg, user=user) + def pkglist_setowner(taginfo, pkginfo, owner, force=False): """Set the owner for package in tag""" pkglist_add(taginfo, pkginfo, owner=owner, force=force, update=True) + def pkglist_setarches(taginfo, pkginfo, arches, force=False): """Set extra_arches for package in tag""" pkglist_add(taginfo, pkginfo, extra_arches=arches, force=force, update=True) + def readPackageList(tagID=None, userID=None, pkgID=None, event=None, inherit=False, with_dups=False): """Returns the package list for the specified tag or user. @@ -1200,6 +1218,7 @@ def readPackageList(tagID=None, userID=None, pkgID=None, event=None, inherit=Fal packages[pkgid] = p return packages + def list_tags(build=None, package=None, perms=True, queryOpts=None): """List tags. If build is specified, only return tags associated with the given build. If package is specified, only return tags associated with the @@ -1266,6 +1285,7 @@ def list_tags(build=None, package=None, perms=True, queryOpts=None): opts=queryOpts) return query.iterate() + def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, owner=None, type=None): """Returns a list of builds for specified tag @@ -1372,6 +1392,7 @@ def readTaggedBuilds(tag, event=None, inherit=False, latest=False, package=None, return builds + def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, latest=True, rpmsigs=False, owner=None, type=None): """Returns a list of rpms for specified tag @@ -1437,6 +1458,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late # duplicate rpminfo entries, BUT since we make the query multiple times, # we can get duplicates if a package is multiply tagged. tags_seen = {} + def _iter_rpms(): for tagid in taglist: if tagid in tags_seen: @@ -1464,6 +1486,7 @@ def readTaggedRPMS(tag, package=None, arch=None, event=None, inherit=False, late yield rpminfo return [_iter_rpms(), builds] + def readTaggedArchives(tag, package=None, event=None, inherit=False, latest=True, type=None): """Returns a list of archives for specified tag @@ -1559,6 +1582,7 @@ def readTaggedArchives(tag, package=None, event=None, inherit=False, latest=True archives.append(archiveinfo) return [archives, builds] + def check_tag_access(tag_id, user_id=None): """Determine if user has access to tag package with tag. @@ -1584,11 +1608,13 @@ def check_tag_access(tag_id, user_id=None): return (False, override, "tag requires %s permission" % needed_perm) return (True, override, "") + def assert_tag_access(tag_id, user_id=None, force=False): access, override, reason = check_tag_access(tag_id, user_id) if not access and not (override and force): raise koji.ActionNotAllowed(reason) + def _tag_build(tag, build, user_id=None, force=False): """Tag a build @@ -2157,6 +2183,7 @@ def get_tag_groups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True return groups + def readTagGroups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True, incl_blocked=False): """Return group data for the tag with blocked entries removed @@ -2184,6 +2211,7 @@ def readTagGroups(tag, event=None, inherit=True, incl_pkgs=True, incl_reqs=True, else: return [x for x in groups if not x['blocked']] + def set_host_enabled(hostname, enabled=True): context.session.assertPerm('host') host = get_host(hostname) @@ -2200,6 +2228,7 @@ def set_host_enabled(hostname, enabled=True): insert.make_create() insert.execute() + def add_host_to_channel(hostname, channel_name, create=False): """Add the host to the specified channel @@ -2222,6 +2251,7 @@ def add_host_to_channel(hostname, channel_name, create=False): insert.make_create() insert.execute() + def remove_host_from_channel(hostname, channel_name): context.session.assertPerm('host') host = get_host(hostname) @@ -2260,6 +2290,7 @@ def rename_channel(old, new): update.set(name=new) update.execute() + def remove_channel(channel_name, force=False): """Remove a channel @@ -2288,6 +2319,7 @@ def remove_channel(channel_name, force=False): delete = """DELETE FROM channels WHERE id=%(channel_id)i""" _dml(delete, locals()) + def get_ready_hosts(): """Return information about hosts that are ready to build. @@ -2317,6 +2349,7 @@ def get_ready_hosts(): host['channels'] = [row[0] for row in c.fetchall()] return hosts + def get_all_arches(): """Return a list of all (canonical) arches available from hosts""" ret = {} @@ -2329,6 +2362,7 @@ def get_all_arches(): ret[koji.canonArch(arch)] = 1 return to_list(ret.keys()) + def get_active_tasks(host=None): """Return data on tasks that are yet to be run""" fields = ['id', 'state', 'channel_id', 'host_id', 'arch', 'method', 'priority', 'create_time'] @@ -2349,6 +2383,7 @@ AND channel_id IN %(channels)s)''' values=values, opts=queryOpts) return query.execute() + def get_task_descendents(task, childMap=None, request=False): if childMap == None: childMap = {} @@ -2360,6 +2395,7 @@ def get_task_descendents(task, childMap=None, request=False): get_task_descendents(Task(child['id']), childMap, request) return childMap + def maven_tag_archives(tag_id, event_id=None, inherit=True): """ Get Maven artifacts associated with the given tag, following inheritance. @@ -2408,6 +2444,7 @@ def maven_tag_archives(tag_id, event_id=None, inherit=True): # group_id/artifact_id/version/build_id/archive_id, which is much smaller than # the full query # ballpark estimate: 20-25% of total, less with heavy duplication of indexed values + def _iter_archives(): for tag_id in taglist: taginfo = get_tag(tag_id, strict=True, event=event_id) @@ -2454,6 +2491,7 @@ def maven_tag_archives(tag_id, event_id=None, inherit=True): yield archive return _iter_archives() + def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separate_src=False): """Create a new repo entry in the INIT state, return full repo data @@ -2605,6 +2643,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None, with_separa event=event, repo_id=repo_id, with_separate_src=with_separate_src) return [repo_id, event_id] + def _write_maven_repo_metadata(destdir, artifacts): # Sort the list so that the highest version number comes last. # group_id and artifact_id should be the same for all entries, @@ -2637,6 +2676,7 @@ def _write_maven_repo_metadata(destdir, artifacts): mdfile.write(contents) _generate_maven_metadata(destdir) + def dist_repo_init(tag, keys, task_opts): """Create a new repo entry in the INIT state, return full repo data""" state = koji.REPO_INIT @@ -2705,6 +2745,7 @@ def repo_set_state(repo_id, state, check=True): q = """UPDATE repo SET state=%(state)s WHERE id = %(repo_id)s""" _dml(q, locals()) + def repo_info(repo_id, strict=False): fields = ( ('repo.id', 'id'), @@ -2722,18 +2763,22 @@ def repo_info(repo_id, strict=False): WHERE repo.id = %%(repo_id)s""" % ','.join([f[0] for f in fields]) return _singleRow(q, locals(), [f[1] for f in fields], strict=strict) + def repo_ready(repo_id): """Set repo state to ready""" repo_set_state(repo_id, koji.REPO_READY) + def repo_expire(repo_id): """Set repo state to expired""" repo_set_state(repo_id, koji.REPO_EXPIRED) + def repo_problem(repo_id): """Set repo state to problem""" repo_set_state(repo_id, koji.REPO_PROBLEM) + def repo_delete(repo_id): """Attempt to mark repo deleted, return number of references @@ -2848,6 +2893,7 @@ def tag_changed_since_event(event, taglist): return True return False + def set_tag_update(tag_id, utype, event_id=None, user_id=None): """Record a non-versioned tag update""" utype_id = koji.TAG_UPDATE_TYPES.getnum(utype) @@ -2863,6 +2909,7 @@ def set_tag_update(tag_id, utype, event_id=None, user_id=None): insert = InsertProcessor('tag_updates', data=data) insert.execute() + def _validate_build_target_name(name): """ A helper function that validates a build target name. """ max_name_length = 256 @@ -3016,6 +3063,7 @@ def get_build_targets(info=None, event=None, buildTagID=None, destTagID=None, qu values=locals(), opts=queryOpts) return query.execute() + def get_build_target(info, event=None, strict=False): """Return the build target with the given name or ID. If there is no matching build target, return None.""" @@ -3027,6 +3075,7 @@ def get_build_target(info, event=None, strict=False): else: return None + def lookup_name(table, info, strict=False, create=False): """Find the id and name in the table associated with info. @@ -3069,6 +3118,7 @@ def lookup_name(table, info, strict=False, create=False): return ret return ret + def get_id(table, info, strict=False, create=False): """Find the id in the table associated with info.""" data = lookup_name(table, info, strict, create) @@ -3077,50 +3127,62 @@ def get_id(table, info, strict=False, create=False): else: return data['id'] + def get_tag_id(info, strict=False, create=False): """Get the id for tag""" return get_id('tag', info, strict, create) + def lookup_tag(info, strict=False, create=False): """Get the id,name for tag""" return lookup_name('tag', info, strict, create) + def get_perm_id(info, strict=False, create=False): """Get the id for a permission""" return get_id('permissions', info, strict, create) + def lookup_perm(info, strict=False, create=False): """Get the id,name for perm""" return lookup_name('permissions', info, strict, create) + def get_package_id(info, strict=False, create=False): """Get the id for a package""" return get_id('package', info, strict, create) + def lookup_package(info, strict=False, create=False): """Get the id,name for package""" return lookup_name('package', info, strict, create) + def get_channel_id(info, strict=False, create=False): """Get the id for a channel""" return get_id('channels', info, strict, create) + def lookup_channel(info, strict=False, create=False): """Get the id,name for channel""" return lookup_name('channels', info, strict, create) + def get_group_id(info, strict=False, create=False): """Get the id for a group""" return get_id('groups', info, strict, create) + def lookup_group(info, strict=False, create=False): """Get the id,name for group""" return lookup_name('groups', info, strict, create) + def get_build_target_id(info, strict=False, create=False): """Get the id for a build target""" return get_id('build_target', info, strict, create) + def lookup_build_target(info, strict=False, create=False): """Get the id,name for build target""" return lookup_name('build_target', info, strict, create) @@ -3190,6 +3252,7 @@ def _create_tag(name, parent=None, arches=None, perm=None, locked=False, maven_s return tag_id + def get_tag(tagInfo, strict=False, event=None): """Get tag information based on the tagInfo. tagInfo may be either a string (the tag name) or an int (the tag ID). @@ -3432,10 +3495,12 @@ def _delete_tag(tagInfo): # is still referenced by the revoked rows). # note: there is no need to do anything with the repo entries that reference tagID + def get_external_repo_id(info, strict=False, create=False): """Get the id for a build target""" return get_id('external_repo', info, strict, create) + def create_external_repo(name, url): """Create a new external repo with the given name and url. Return a map containing the id, name, and url @@ -3457,6 +3522,7 @@ def create_external_repo(name, url): insert.execute() return values + def get_external_repos(info=None, url=None, event=None, queryOpts=None): """Get a list of external repos. If info is not None it may be a string (name) or an integer (id). @@ -3481,6 +3547,7 @@ def get_external_repos(info=None, url=None, event=None, queryOpts=None): values=locals(), opts=queryOpts) return query.execute() + def get_external_repo(info, strict=False, event=None): """Get information about a single external repo. info can either be a string (name) or an integer (id). @@ -3496,6 +3563,7 @@ def get_external_repo(info, strict=False, event=None): else: return None + def edit_external_repo(info, name=None, url=None): """Edit an existing external repo""" @@ -3529,6 +3597,7 @@ def edit_external_repo(info, name=None, url=None): update.execute() insert.execute() + def delete_external_repo(info): """Delete an external repo""" @@ -3546,6 +3615,7 @@ def delete_external_repo(info): update.make_revoke() update.execute() + def add_external_repo_to_tag(tag_info, repo_info, priority, merge_mode='koji'): """Add an external repo to a tag""" @@ -3573,6 +3643,7 @@ def add_external_repo_to_tag(tag_info, repo_info, priority, merge_mode='koji'): insert.make_create() insert.execute() + def remove_external_repo_from_tag(tag_info, repo_info): """Remove an external repo from a tag""" @@ -3592,6 +3663,7 @@ def remove_external_repo_from_tag(tag_info, repo_info): update.make_revoke() update.execute() + def edit_tag_external_repo(tag_info, repo_info, priority): """Edit a tag<->external repo association This allows you to update the priority without removing/adding the repo.""" @@ -3613,6 +3685,7 @@ def edit_tag_external_repo(tag_info, repo_info, priority): remove_external_repo_from_tag(tag_id, repo_id) add_external_repo_to_tag(tag_id, repo_id, priority) + def get_tag_external_repos(tag_info=None, repo_info=None, event=None): """ Get a list of tag<->external repo associations. @@ -3659,6 +3732,7 @@ def get_tag_external_repos(tag_info=None, repo_info=None, event=None): opts=opts) return query.execute() + def get_external_repo_list(tag_info, event=None): """ Get an ordered list of all external repos associated with the tags in the @@ -3757,6 +3831,7 @@ def get_user(userInfo=None, strict=False, krb_princs=True): user['krb_principals'] = list_user_krb_principals(user['id']) return user + def edit_user(userInfo, name=None, krb_principal_mappings=None): """Edit information for an existing user. @@ -4068,6 +4143,7 @@ def _fix_rpm_row(row): row['extra'] = parse_json(row['extra'], desc='rpm extra') return row + # alias for now, may change in the future _fix_archive_row = _fix_rpm_row @@ -4275,6 +4351,7 @@ def get_maven_build(buildInfo, strict=False): WHERE build_id = %%(build_id)i""" % ', '.join(fields) return _singleRow(query, locals(), fields, strict) + def get_win_build(buildInfo, strict=False): """ Retrieve Windows-specific information about a build. @@ -4298,6 +4375,7 @@ def get_win_build(buildInfo, strict=False): raise koji.GenericError('no such Windows build: %s' % buildInfo) return result + def get_image_build(buildInfo, strict=False): """ Retrieve image-specific information about a build. @@ -4629,6 +4707,7 @@ def get_archive(archive_id, strict=False): archive.update(image_info) return archive + def get_maven_archive(archive_id, strict=False): """ Retrieve Maven-specific information about an archive. @@ -4644,6 +4723,7 @@ def get_maven_archive(archive_id, strict=False): WHERE archive_id = %%(archive_id)i""" % ', '.join(fields) return _singleRow(select, locals(), fields, strict=strict) + def get_win_archive(archive_id, strict=False): """ Retrieve Windows-specific information about an archive. @@ -4659,6 +4739,7 @@ def get_win_archive(archive_id, strict=False): WHERE archive_id = %%(archive_id)i""" % ', '.join(fields) return _singleRow(select, locals(), fields, strict=strict) + def get_image_archive(archive_id, strict=False): """ Retrieve image-specific information about an archive. @@ -4683,6 +4764,7 @@ def get_image_archive(archive_id, strict=False): results['rootid'] = True return results + def _get_zipfile_list(archive_id, zippath): """ Get a list of the entries in the zipfile located at zippath. @@ -4704,6 +4786,7 @@ def _get_zipfile_list(archive_id, zippath): 'mtime': int(time.mktime(entry.date_time + (0, 0, -1)))}) return result + def _get_tarball_list(archive_id, tarpath): """ Get a list of the entries in the tarball located at tarpath. @@ -4891,6 +4974,7 @@ def list_task_output(taskID, stat=False, all_volumes=False, strict=False): result.append(relfilename) return result + def _fetchMulti(query, values): """Run the query and return all rows""" c = context.cnx.cursor() @@ -4899,6 +4983,7 @@ def _fetchMulti(query, values): c.close() return results + def _fetchSingle(query, values, strict=False): """Run the query and return a single row @@ -4916,6 +5001,7 @@ def _fetchSingle(query, values, strict=False): else: return results[0] + def _multiRow(query, values, fields): """Return all rows from "query". Named query parameters can be specified using the "values" map. Results will be returned @@ -4924,6 +5010,7 @@ def _multiRow(query, values, fields): list will be returned.""" return [dict(zip(fields, row)) for row in _fetchMulti(query, values)] + def _singleRow(query, values, fields, strict=False): """Return a single row from "query". Named parameters can be specified using the "values" map. The result will be returned as @@ -4939,6 +5026,7 @@ def _singleRow(query, values, fields, strict=False): # strict enforced by _fetchSingle return None + def _singleValue(query, values=None, strict=True): """Perform a query that returns a single value. @@ -4955,6 +5043,7 @@ def _singleValue(query, values=None, strict=True): # don't need to check strict here, since that was already handled by _singleRow() return None + def _dml(operation, values): """Run an insert, update, or delete. Return number of rows affected""" c = context.cnx.cursor() @@ -4965,6 +5054,7 @@ def _dml(operation, values): context.commit_pending = True return ret + def get_host(hostInfo, strict=False, event=None): """Get information about the given host. hostInfo may be either a string (hostname) or int (host id). A map will be returned @@ -5015,6 +5105,7 @@ def get_host(hostInfo, strict=False, event=None): return None return result + def edit_host(hostInfo, **kw): """Edit information for an existing host. hostInfo specifies the host to edit, either as an integer (id) @@ -5053,6 +5144,7 @@ def edit_host(hostInfo, **kw): return True + def get_channel(channelInfo, strict=False): """ Look up the ID number and name for a channel. @@ -5155,6 +5247,7 @@ def query_buildroots(hostID=None, tagID=None, state=None, rpmID=None, archiveID= opts=queryOpts) return query.execute() + def get_buildroot(buildrootID, strict=False): """Return information about a buildroot. buildrootID must be an int ID.""" @@ -5169,6 +5262,7 @@ def get_buildroot(buildrootID, strict=False): raise koji.GenericError("More that one buildroot with id: %i" % buildrootID) return result[0] + def list_channels(hostID=None, event=None): """List channels. If hostID is specified, only list channels associated with the host with that ID.""" @@ -5192,6 +5286,7 @@ def list_channels(hostID=None, event=None): columns=columns) return query.execute() + def new_package(name, strict=True): c = context.cnx.cursor() # TODO - table lock? @@ -5226,6 +5321,7 @@ def add_volume(name, strict=True): volinfo = lookup_name('volume', name, strict=False, create=True) return volinfo + def remove_volume(volume): """Remove unused storage volume from the database""" context.session.assertPerm('admin') @@ -5237,10 +5333,12 @@ def remove_volume(volume): delete = """DELETE FROM volume WHERE id=%(id)i""" _dml(delete, volinfo) + def list_volumes(): """List storage volumes""" return QueryProcessor(tables=['volume'], columns=['id', 'name']).execute() + def change_build_volume(build, volume, strict=True): """Move a build to a different storage volume""" context.session.assertPerm('admin') @@ -5598,6 +5696,7 @@ def check_noarch_rpms(basepath, rpms, logs=None): return result + def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None): """Import a build into the database (single transaction) @@ -5962,7 +6061,6 @@ class CG_Importer(object): return self.buildinfo - def get_metadata(self, metadata, directory): """Get the metadata from the args""" @@ -5991,7 +6089,6 @@ class CG_Importer(object): self.metadata = parse_json(metadata, desc='metadata') return self.metadata - def assert_cg_access(self): """Check that user has access for all referenced content generators""" @@ -6005,7 +6102,6 @@ class CG_Importer(object): assert_cg(cg_id) self.cgs = cgs - def assert_policy(self): policy_data = { 'package': self.buildinfo['name'], @@ -6016,7 +6112,6 @@ class CG_Importer(object): } assert_policy('cg_import', policy_data) - def set_volume(self): """Use policy to determine what the volume should be""" # we have to be careful and provide sufficient data @@ -6034,7 +6129,6 @@ class CG_Importer(object): self.buildinfo['volume_id'] = vol['id'] self.buildinfo['volume_name'] = vol['name'] - def check_build_dir(self, delete=False): """Check that the import directory does not already exist""" path = koji.pathinfo.build(self.buildinfo) @@ -6045,8 +6139,6 @@ class CG_Importer(object): else: raise koji.GenericError("Destination directory already exists: %s" % path) - - def prep_build(self, token=None): metadata = self.metadata if metadata['build'].get('build_id'): @@ -6120,7 +6212,6 @@ class CG_Importer(object): self.typeinfo = typeinfo return buildinfo - def get_build(self, token=None): if token: # token and reservation were already checked in prep_build @@ -6153,7 +6244,6 @@ class CG_Importer(object): self.buildinfo = buildinfo return buildinfo - def update_build(self): """Update a reserved build""" # sanity checks performed by prep_build @@ -6181,7 +6271,6 @@ class CG_Importer(object): return buildinfo - def import_metadata(self): """Import the raw metadata""" @@ -6195,7 +6284,6 @@ class CG_Importer(object): with open(path, 'w') as fo: fo.write(self.raw_metadata) - def prep_brs(self): metadata = self.metadata br_used = set([f['buildroot_id'] for f in metadata['output']]) @@ -6209,7 +6297,6 @@ class CG_Importer(object): br_idx[brfakeid] = self.prep_buildroot(brdata) self.br_prep = br_idx - def import_brs(self): brmap = {} for brfakeid in self.br_prep: @@ -6217,7 +6304,6 @@ class CG_Importer(object): brmap[brfakeid] = self.import_buildroot(entry) self.brmap = brmap - def prep_buildroot(self, brdata): ret = {} brinfo = { @@ -6238,7 +6324,6 @@ class CG_Importer(object): } return ret - def import_buildroot(self, entry): """Import the prepared buildroot data""" @@ -6255,7 +6340,6 @@ class CG_Importer(object): return br - def match_components(self, components): rpms = [] files = [] @@ -6276,7 +6360,6 @@ class CG_Importer(object): raise koji.GenericError("Unknown component type: %(type)s" % comp) return rpms, files - def match_rpm(self, comp): # TODO: do we allow inclusion of external rpms? if 'location' in comp: @@ -6298,7 +6381,6 @@ class CG_Importer(object): # TODO - should we check the signature field? return rinfo - def match_file(self, comp): # hmm, how do we look up archives? # updateMavenBuildRootList does seriously wild stuff @@ -6389,7 +6471,6 @@ class CG_Importer(object): outputs.append(fileinfo) self.prepped_outputs = outputs - def import_outputs(self): for fileinfo in self.prepped_outputs: brinfo = self.brmap.get(fileinfo['buildroot_id']) @@ -6405,7 +6486,6 @@ class CG_Importer(object): self.import_archive(self.buildinfo, brinfo, fileinfo) ensure_volume_symlink(self.buildinfo) - def prep_archive(self, fileinfo): # determine archive import type extra = fileinfo.get('extra', {}) @@ -6450,7 +6530,6 @@ class CG_Importer(object): fileinfo['hub.rpmlist'] = rpmlist fileinfo['hub.archives'] = archives - def import_rpm(self, buildinfo, brinfo, fileinfo): if fileinfo.get('metadata_only', False): raise koji.GenericError('Metadata-only imports are not supported for rpms') @@ -6460,7 +6539,6 @@ class CG_Importer(object): import_rpm_file(fn, buildinfo, rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) - def import_log(self, buildinfo, fileinfo): if fileinfo.get('metadata_only', False): # logs are not currently tracked, so this is a no op @@ -6469,7 +6547,6 @@ class CG_Importer(object): fn = fileinfo['hub.path'] import_build_log(fn, buildinfo, subdir=None) - def import_archive(self, buildinfo, brinfo, fileinfo): fn = fileinfo['hub.path'] btype = fileinfo['hub.btype'] @@ -6480,7 +6557,6 @@ class CG_Importer(object): if 'components' in fileinfo: self.import_components(archiveinfo['id'], fileinfo) - def import_components(self, archive_id, fileinfo): rpmlist = fileinfo['hub.rpmlist'] archives = fileinfo['hub.archives'] @@ -6574,6 +6650,7 @@ def add_external_rpm(rpminfo, external_repo, strict=True): return get_rpm(data['id']) + def import_build_log(fn, buildinfo, subdir=None): """Move a logfile related to a build to the right place""" logdir = koji.pathinfo.build_logs(buildinfo) @@ -6587,6 +6664,7 @@ def import_build_log(fn, buildinfo, subdir=None): raise koji.GenericError("Error importing build log. %s is not a regular file." % fn) move_and_symlink(fn, final_path) + def import_rpm_file(fn, buildinfo, rpminfo): """Move the rpm file into the proper place @@ -6595,6 +6673,7 @@ def import_rpm_file(fn, buildinfo, rpminfo): final_path = "%s/%s" % (koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(rpminfo)) _import_archive_file(fn, os.path.dirname(final_path)) + def _import_wrapper(task_id, build_info, rpm_results): """Helper function to import wrapper rpms for a Maven build""" rpm_buildroot_id = rpm_results['buildroot_id'] @@ -6611,6 +6690,7 @@ def _import_wrapper(task_id, build_info, rpm_results): import_build_log(joinpath(rpm_task_dir, log), build_info, subdir='noarch') + def merge_scratch(task_id): """Import rpms from a scratch build into an existing build, retaining buildroot metadata and build logs.""" @@ -6724,22 +6804,26 @@ def merge_scratch(task_id): return build['id'] + def get_archive_types(): """Return a list of all supported archive types.""" select = """SELECT id, name, description, extensions FROM archivetypes ORDER BY id""" return _multiRow(select, {}, ('id', 'name', 'description', 'extensions')) + def _get_archive_type_by_name(name, strict=True): select = """SELECT id, name, description, extensions FROM archivetypes WHERE name = %(name)s""" return _singleRow(select, locals(), ('id', 'name', 'description', 'extensions'), strict) + def _get_archive_type_by_id(type_id, strict=False): select = """SELECT id, name, description, extensions FROM archivetypes WHERE id = %(type_id)i""" return _singleRow(select, locals(), ('id', 'name', 'description', 'extensions'), strict) + def get_archive_type(filename=None, type_name=None, type_id=None, strict=False): """ Get the archive type for the given filename, type_name, or type_id. @@ -6776,6 +6860,7 @@ def get_archive_type(filename=None, type_name=None, type_id=None, strict=False): else: return None + def add_archive_type(name, description, extensions): """ Add new archive type. @@ -6830,6 +6915,7 @@ def new_maven_build(build, maven_info): # also add build_types entry new_typed_build(build, 'maven') + def new_win_build(build_info, win_info): """ Add Windows metadata to an existing build. @@ -6851,6 +6937,7 @@ def new_win_build(build_info, win_info): # also add build_types entry new_typed_build(build_info, 'win') + def new_image_build(build_info): """ Added Image metadata to an existing build. This is just the buildid so that @@ -7067,6 +7154,7 @@ def _import_archive_file(filepath, destdir): raise koji.GenericError("Error importing archive file, %s is not a regular file" % filepath) move_and_symlink(filepath, final_path, create_dir=True) + def _generate_maven_metadata(mavendir): """ Generate md5 and sha1 sums for every file in mavendir, if it doesn't already exist. @@ -7091,6 +7179,7 @@ def _generate_maven_metadata(mavendir): with open('%s/%s' % (mavendir, sumfile), 'w') as sumobj: sumobj.write(sum.hexdigest()) + def add_rpm_sig(an_rpm, sighdr): """Store a signature header for an rpm""" # calling function should perform permission checks, if applicable @@ -7147,6 +7236,7 @@ def add_rpm_sig(an_rpm, sighdr): fo.write(sighdr) koji.plugin.run_callbacks('postRPMSign', sigkey=sigkey, sighash=sighash, build=binfo, rpm=rinfo) + def _scan_sighdr(sighdr, fn): """Splices sighdr with other headers from fn and queries (no payload)""" # This is hackish, but it works @@ -7179,6 +7269,7 @@ def _scan_sighdr(sighdr, fn): sig = koji.get_header_field(hdr, 'sigpgp') return koji.get_header_field(hdr, 'sigmd5'), sig + def check_rpm_sig(an_rpm, sigkey, sighdr): # verify that the provided signature header matches the key and rpm rinfo = get_rpm(an_rpm, strict=True) @@ -7216,7 +7307,6 @@ def check_rpm_sig(an_rpm, sigkey, sighdr): os.unlink(temp) - def query_rpm_sigs(rpm_id=None, sigkey=None, queryOpts=None): fields = ('rpm_id', 'sigkey', 'sighash') clauses = [] @@ -7228,6 +7318,7 @@ def query_rpm_sigs(rpm_id=None, sigkey=None, queryOpts=None): values=locals(), opts=queryOpts) return query.execute() + def write_signed_rpm(an_rpm, sigkey, force=False): """Write a signed copy of the rpm""" rinfo = get_rpm(an_rpm, strict=True) @@ -7599,6 +7690,7 @@ def tag_history(build=None, tag=None, package=None, active=None, queryOpts=None) opts=queryOpts) return query.iterate() + def untagged_builds(name=None, queryOpts=None): """Returns the list of untagged builds""" fields = ('build.id', 'package.name', 'build.version', 'build.release') @@ -7862,6 +7954,7 @@ def _delete_build(binfo): binfo = get_build(build_id, strict=True) koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_deleted, info=binfo) + def reset_build(build): """Reset a build so that it can be reimported @@ -7931,6 +8024,7 @@ def reset_build(build): binfo = get_build(build, strict=True) koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=koji.BUILD_STATES['CANCELED'], info=binfo) + def cancel_build(build_id, cancel_task=True): """Cancel a build @@ -7971,6 +8065,7 @@ def cancel_build(build_id, cancel_task=True): koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=st_old, new=st_canceled, info=build) return True + def _get_build_target(task_id): # XXX Should we be storing a reference to the build target # in the build table for reproducibility? @@ -7987,6 +8082,7 @@ def _get_build_target(task_id): return get_build_target(request[2]) return None + def get_notification_recipients(build, tag_id, state): """ Return the list of email addresses that should be notified about events @@ -8111,6 +8207,7 @@ def tag_notification(is_successful, tag_id, from_id, build_id, user_id, ignore_s return task_id return None + def build_notification(task_id, build_id): if context.opts.get('DisableNotifications'): return @@ -8130,6 +8227,7 @@ def build_notification(task_id, build_id): if len(recipients) > 0: make_task('buildNotification', [recipients, build, target, web_url]) + def get_build_notifications(user_id): query = QueryProcessor(tables=['build_notifications'], columns=('id', 'user_id', 'package_id', 'tag_id', @@ -8138,6 +8236,7 @@ def get_build_notifications(user_id): values=locals()) return query.execute() + def get_build_notification_blocks(user_id): query = QueryProcessor(tables=['build_notifications_block'], columns=['id', 'user_id', 'package_id', 'tag_id'], @@ -8153,6 +8252,7 @@ def new_group(name): raise koji.GenericError('user/group already exists: %s' % name) return context.session.createUser(name, usertype=koji.USERTYPES['GROUP']) + def add_group_member(group, user, strict=True): """Add user to group""" context.session.assertPerm('admin') @@ -8180,6 +8280,7 @@ def add_group_member(group, user, strict=True): insert.make_create() insert.execute() + def drop_group_member(group, user): """Drop user from group""" context.session.assertPerm('admin') @@ -8195,6 +8296,7 @@ def drop_group_member(group, user): update.make_revoke() update.execute() + def get_group_members(group): """Get the members of a group""" context.session.assertPerm('admin') @@ -8220,6 +8322,7 @@ def get_group_members(group): transform=xform_user_krb) return query.iterate() + def set_user_status(user, status): context.session.assertPerm('admin') if not koji.USER_STATUS.get(status): @@ -8813,7 +8916,6 @@ SELECT %(col_str)s # self.transform not applied here return _singleValue(str(self), self.values, strict=strict) - def execute(self): query = str(self) if self.opts.get('countOnly'): @@ -8834,7 +8936,6 @@ SELECT %(col_str)s data = [self.transform(row) for row in data] return data - def iterate(self): if self.opts.get('countOnly'): return self.execute() @@ -8893,6 +8994,7 @@ SELECT %(col_str)s return None return results + def _applyQueryOpts(results, queryOpts): """ Apply queryOpts to results in the same way QueryProcessor would. @@ -8936,6 +9038,7 @@ class OperationTest(koji.policy.MatchTest): name = 'operation' field = 'operation' + def policy_get_user(data): """Determine user from policy data (default to logged-in user)""" if 'user_id' in data: @@ -8944,6 +9047,7 @@ def policy_get_user(data): return get_user(context.session.user_id) return None + def policy_get_pkg(data): """Determine package from policy data (default to logged-in user) @@ -9050,16 +9154,20 @@ def policy_get_build_types(data): return set(get_build_type(binfo).keys()) return set() + class NewPackageTest(koji.policy.BaseSimpleTest): """Checks to see if a package exists yet""" name = 'is_new_package' + def run(self, data): return (policy_get_pkg(data)['id'] is None) + class PackageTest(koji.policy.MatchTest): """Checks package against glob patterns""" name = 'package' field = '_package' + def run(self, data): # we need to find the package name from the base data data[self.field] = policy_get_pkg(data)['name'] @@ -9070,6 +9178,7 @@ class VersionTest(koji.policy.MatchTest): """Checks version against glob patterns""" name = 'version' field = '_version' + def run(self, data): data[self.field] = policy_get_version(data) return super(VersionTest, self).run(data) @@ -9079,6 +9188,7 @@ class ReleaseTest(koji.policy.MatchTest): """Checks release against glob patterns""" name = 'release' field = '_release' + def run(self, data): # we need to find the build NVR from the base data data[self.field] = policy_get_release(data) @@ -9089,6 +9199,7 @@ class VolumeTest(koji.policy.MatchTest): """Checks storage volume against glob patterns""" name = 'volume' field = '_volume' + def run(self, data): # we need to find the volume name from the base data volinfo = None @@ -9172,17 +9283,21 @@ class TagTest(koji.policy.MatchTest): data[self.field] = tinfo['name'] return super(TagTest, self).run(data) + class FromTagTest(TagTest): name = 'fromtag' + def get_tag(self, data): tag = data.get('fromtag') if tag is None: return None return get_tag(tag, strict=False) + class HasTagTest(koji.policy.BaseSimpleTest): """Check to see if build (currently) has a given tag""" name = 'hastag' + def run(self, data): if 'build' not in data: return False @@ -9196,12 +9311,14 @@ class HasTagTest(koji.policy.BaseSimpleTest): # otherwise... return False + class SkipTagTest(koji.policy.BaseSimpleTest): """Check for the skip_tag option For policies regarding build tasks (e.g. build_from_srpm) """ name = 'skip_tag' + def run(self, data): return bool(data.get('skip_tag')) @@ -9213,6 +9330,7 @@ class BuildTagTest(koji.policy.BaseSimpleTest): buildroots of the component rpms """ name = 'buildtag' + def run(self, data): args = self.str.split()[1:] for tagname in policy_get_build_tags(data): @@ -9229,6 +9347,7 @@ class BuildTypeTest(koji.policy.BaseSimpleTest): """Check the build type(s) of the build""" name = 'buildtype' + def run(self, data): args = self.str.split()[1:] for btype in policy_get_build_types(data): @@ -9243,6 +9362,7 @@ class ImportedTest(koji.policy.BaseSimpleTest): This is determined by checking the buildroots of the rpms and archives True if any of them lack a buildroot (strict)""" name = 'imported' + def run(self, data): build_info = data.get('build') if not build_info: @@ -9258,18 +9378,22 @@ class ImportedTest(koji.policy.BaseSimpleTest): # otherwise... return False + class ChildTaskTest(koji.policy.BoolTest): name = 'is_child_task' field = 'parent' + class MethodTest(koji.policy.MatchTest): name = 'method' field = 'method' + class UserTest(koji.policy.MatchTest): """Checks username against glob patterns""" name = 'user' field = '_username' + def run(self, data): user = policy_get_user(data) if not user: @@ -9277,14 +9401,17 @@ class UserTest(koji.policy.MatchTest): data[self.field] = user['name'] return super(UserTest, self).run(data) + class VMTest(koji.policy.MatchTest): """Checks a VM name against glob patterns""" name = 'vm_name' field = 'vm_name' + class IsBuildOwnerTest(koji.policy.BaseSimpleTest): """Check if user owns the build""" name = "is_build_owner" + def run(self, data): build = get_build(data['build']) owner = get_user(build['owner_id']) @@ -9300,6 +9427,7 @@ class IsBuildOwnerTest(koji.policy.BaseSimpleTest): # otherwise... return False + class UserInGroupTest(koji.policy.BaseSimpleTest): """Check if user is in group(s) @@ -9307,6 +9435,7 @@ class UserInGroupTest(koji.policy.BaseSimpleTest): true if user is in /any/ matching group """ name = "user_in_group" + def run(self, data): user = policy_get_user(data) if not user: @@ -9320,6 +9449,7 @@ class UserInGroupTest(koji.policy.BaseSimpleTest): # otherwise... return False + class HasPermTest(koji.policy.BaseSimpleTest): """Check if user has permission(s) @@ -9327,6 +9457,7 @@ class HasPermTest(koji.policy.BaseSimpleTest): true if user has /any/ matching permission """ name = "has_perm" + def run(self, data): user = policy_get_user(data) if not user: @@ -9340,6 +9471,7 @@ class HasPermTest(koji.policy.BaseSimpleTest): # otherwise... return False + class SourceTest(koji.policy.MatchTest): """Match build source @@ -9348,6 +9480,7 @@ class SourceTest(koji.policy.MatchTest): """ name = "source" field = '_source' + def run(self, data): if 'source' in data: data[self.field] = data['source'] @@ -9377,6 +9510,7 @@ class SourceTest(koji.policy.MatchTest): return False return super(SourceTest, self).run(data) + class PolicyTest(koji.policy.BaseSimpleTest): """Test named policy @@ -9461,6 +9595,7 @@ def check_policy(name, data, default='deny', strict=False): err_str += " [rule: %s]" % lastrule raise koji.ActionNotAllowed(err_str) + def assert_policy(name, data, default='deny'): """Enforce the named policy @@ -9471,6 +9606,7 @@ def assert_policy(name, data, default='deny'): """ check_policy(name, data, default=default, strict=True) + def rpmdiff(basepath, rpmlist, hashes): "Diff the first rpm in the list against the rest of the rpms." if len(rpmlist) < 2: @@ -9580,6 +9716,8 @@ def importImageInternal(task_id, build_id, imgdata): # # XMLRPC Methods # + + class RootExports(object): '''Contains functions that are made available via XMLRPC''' @@ -10067,7 +10205,6 @@ class RootExports(object): # this will also free our lock os.close(fd) - def downloadTaskOutput(self, taskID, fileName, offset=0, size=-1, volume=None): """Download the file with the given name, generated by the task with the given ID.""" @@ -10088,7 +10225,6 @@ class RootExports(object): contents = f.read(size) return base64encode(contents) - listTaskOutput = staticmethod(list_task_output) createTag = staticmethod(create_tag) @@ -10155,6 +10291,7 @@ class RootExports(object): queryHistory = staticmethod(query_history) deleteBuild = staticmethod(delete_build) + def buildReferences(self, build, limit=None, lazy=False): return build_references(get_build(build, strict=True)['id'], limit, lazy) @@ -10474,7 +10611,6 @@ class RootExports(object): tasklist.append(task_id) return tasklist - listTags = staticmethod(list_tags) getBuild = staticmethod(get_build) @@ -10600,7 +10736,6 @@ class RootExports(object): results = _applyQueryOpts(results, queryOpts) return koji.fixEncodingRecurse(results, remove_nonprintable=True) - def cancelBuild(self, buildID): """Cancel the build with the given buildID @@ -11258,7 +11393,6 @@ class RootExports(object): return _applyQueryOpts(results, queryOpts) - def listPackagesSimple(self, prefix=None, queryOpts=None): """list packages that starts with prefix and are filted and ordered by queryOpts. @@ -11284,7 +11418,6 @@ class RootExports(object): opts=queryOpts) return query.execute() - def checkTagPackage(self, tag, pkg): """Check that pkg is in the list for tag. Returns true/false""" tag_id = get_tag_id(tag, strict=False) @@ -12090,7 +12223,6 @@ class RootExports(object): """ return self.countAndFilterResults(methodName, *args, **kw)[1] - def countAndFilterResults(self, methodName, *args, **kw): """Filter results by a given name and count total results account. @@ -12146,7 +12278,6 @@ class RootExports(object): return _count, results - def getBuildNotifications(self, userID=None): """Get build notifications for the user with the given ID, name or Kerberos principal. If no user is specified, get the notifications for @@ -12760,7 +12891,6 @@ class Host(object): else: logger.warning('taskSetWait called on empty task list by parent: %s', parent) - def taskWaitCheck(self, parent): """Return status of awaited subtask @@ -12925,6 +13055,7 @@ class Host(object): query = """SELECT enabled FROM host_config WHERE host_id = %(id)i AND active IS TRUE""" return _singleValue(query, {'id': self.id}, strict=True) + class HostExports(object): '''Contains functions that are made available via XMLRPC''' @@ -13222,7 +13353,6 @@ class HostExports(object): build_info['volume_name'] = vol['name'] vol_update = True - self.importImage(task_id, build_id, results) ensure_volume_symlink(build_info) @@ -13890,7 +14020,6 @@ class HostExports(object): log_error("Unable to create latest link for repo: %s" % repodir) koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire) - def distRepoMove(self, repo_id, uploadpath, arch): """ Move one arch of a dist repo into its final location @@ -14002,7 +14131,6 @@ class HostExports(object): else: raise - def isEnabled(self): host = Host() host.verify() @@ -14051,6 +14179,7 @@ def get_upload_path(reldir, name, create=False, volume=None): fo.write(str(context.session.user_id)) return joinpath(udir, name) + def get_verify_class(verify): if verify == 'md5': return hashlib.md5 diff --git a/hub/kojixmlrpc.py b/hub/kojixmlrpc.py index bdd23cb9..ea28d0cc 100644 --- a/hub/kojixmlrpc.py +++ b/hub/kojixmlrpc.py @@ -379,6 +379,7 @@ def offline_reply(start_response, msg=None): start_response('200 OK', headers) return [response] + def load_config(environ): """Load configuration options @@ -509,6 +510,7 @@ def load_plugins(opts): opts['OfflineMessage'] = 'configuration error' return tracker + _default_policies = { 'build_from_srpm': ''' has_perm admin :: allow @@ -539,6 +541,7 @@ _default_policies = { ''', } + def get_policy(opts, plugins): if not opts.get('policy'): return @@ -593,6 +596,7 @@ class HubFormatter(logging.Formatter): record.user_name = None return logging.Formatter.format(self, record) + def setup_logging1(): """Set up basic logging, before options are loaded""" global log_handler @@ -605,6 +609,7 @@ def setup_logging1(): log_handler.setLevel(logging.DEBUG) logger.addHandler(log_handler) + def setup_logging2(opts): global log_handler """Adjust logging based on configuration options""" @@ -659,6 +664,7 @@ def get_memory_usage(): size, res, shr, text, lib, data, dirty = statm return res - shr + def server_setup(environ): global opts, plugins, registry, policy logger = logging.getLogger('koji') @@ -692,6 +698,7 @@ def server_setup(environ): firstcall = True firstcall_lock = threading.Lock() + def application(environ, start_response): global firstcall if firstcall: diff --git a/koji/__init__.py b/koji/__init__.py index 65de6ad0..20cb3be9 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -91,12 +91,14 @@ except ImportError: PROFILE_MODULES = {} # {module_name: module_instance} + def _(args): """Stub function for translation""" return args # pragma: no cover ## Constants ## + RPM_HEADER_MAGIC = six.b('\x8e\xad\xe8') RPM_TAG_HEADERSIGNATURES = 62 RPM_TAG_FILEDIGESTALGO = 5011 @@ -130,6 +132,7 @@ for h in ( # BEGIN kojikamid dup # + class Enum(dict): """A simple class to track our enumerated constants @@ -178,6 +181,7 @@ class Enum(dict): # END kojikamid dup # + API_VERSION = 1 TASK_STATES = Enum(( @@ -290,10 +294,12 @@ DEFAULT_AUTH_TIMEOUT = 60 # Exceptions PythonImportError = ImportError # will be masked by koji's one + class GenericError(Exception): """Base class for our custom exceptions""" faultCode = 1000 fromFault = False + def __str__(self): try: return str(self.args[0]['args'][0]) @@ -304,97 +310,120 @@ class GenericError(Exception): return str(self.__dict__) # END kojikamid dup # + class LockError(GenericError): """Raised when there is a lock conflict""" faultCode = 1001 + class AuthError(GenericError): """Raised when there is an error in authentication""" faultCode = 1002 + class TagError(GenericError): """Raised when a tagging operation fails""" faultCode = 1003 + class ActionNotAllowed(GenericError): """Raised when the session does not have permission to take some action""" faultCode = 1004 # BEGIN kojikamid dup # + class BuildError(GenericError): """Raised when a build fails""" faultCode = 1005 # END kojikamid dup # + class AuthLockError(AuthError): """Raised when a lock prevents authentication""" faultCode = 1006 + class AuthExpired(AuthError): """Raised when a session has expired""" faultCode = 1007 + class SequenceError(AuthError): """Raised when requests are received out of sequence""" faultCode = 1008 + class RetryError(AuthError): """Raised when a request is received twice and cannot be rerun""" faultCode = 1009 + class PreBuildError(BuildError): """Raised when a build fails during pre-checks""" faultCode = 1010 + class PostBuildError(BuildError): """Raised when a build fails during post-checks""" faultCode = 1011 + class BuildrootError(BuildError): """Raised when there is an error with the buildroot""" faultCode = 1012 + class FunctionDeprecated(GenericError): """Raised by a deprecated function""" faultCode = 1013 + class ServerOffline(GenericError): """Raised when the server is offline""" faultCode = 1014 + class LiveCDError(GenericError): """Raised when LiveCD Image creation fails""" faultCode = 1015 + class PluginError(GenericError): """Raised when there is an error with a plugin""" faultCode = 1016 + class CallbackError(PluginError): """Raised when there is an error executing a callback""" faultCode = 1017 + class ApplianceError(GenericError): """Raised when Appliance Image creation fails""" faultCode = 1018 + class ParameterError(GenericError): """Raised when an rpc call receives incorrect arguments""" faultCode = 1019 + class ImportError(GenericError): """Raised when an import fails""" faultCode = 1020 + class ConfigurationError(GenericError): """Raised when load of koji configuration fails""" faultCode = 1021 + class LiveMediaError(GenericError): """Raised when LiveMedia Image creation fails""" faultCode = 1022 + class MultiCallInProgress(object): """ Placeholder class to be returned by method calls when in the process of @@ -418,6 +447,7 @@ def convertFault(fault): # otherwise... return fault + def listFaults(): """Return a list of faults @@ -442,6 +472,7 @@ def listFaults(): # functions for encoding/decoding optional arguments + def encode_args(*args, **opts): """The function encodes optional arguments as regular arguments. @@ -453,6 +484,7 @@ def encode_args(*args, **opts): args = args + (opts,) return args + def decode_args(*args): """Decodes optional arguments from a flat argument list @@ -468,6 +500,7 @@ def decode_args(*args): args = args[:-1] return args, opts + def decode_args2(args, names, strict=True): "An alternate form of decode_args, returns a dictionary" args, opts = decode_args(*args) @@ -477,6 +510,7 @@ def decode_args2(args, names, strict=True): ret.update(opts) return ret + def decode_int(n): """If n is not an integer, attempt to convert it""" if isinstance(n, six.integer_types): @@ -486,6 +520,7 @@ def decode_int(n): # commonly used functions + def safe_xmlrpc_loads(s): """Load xmlrpc data from a string, but catch faults""" try: @@ -530,6 +565,7 @@ def ensuredir(directory): # END kojikamid dup # + def daemonize(): """Detach and run in background""" pid = os.fork() @@ -553,6 +589,7 @@ def daemonize(): os.close(fd1) os.close(fd2) + def multibyte(data): """Convert a list of bytes to an integer (network byte order)""" sum = 0 @@ -561,6 +598,7 @@ def multibyte(data): sum += data[i] << (8 * (n - i - 1)) return sum + def find_rpm_sighdr(path): """Finds the offset and length of the signature header.""" # see Maximum RPM Appendix A: Format of the RPM File @@ -570,6 +608,7 @@ def find_rpm_sighdr(path): sigsize = rpm_hdr_size(path, sig_start) return (sig_start, sigsize) + def rpm_hdr_size(f, ofs=None): """Returns the length (in bytes) of the rpm header @@ -776,6 +815,7 @@ def rip_rpm_sighdr(src): fo.close() return sighdr + def rip_rpm_hdr(src): """Rip the main header out of an rpm""" (start, size) = find_rpm_sighdr(src) @@ -787,6 +827,7 @@ def rip_rpm_hdr(src): fo.close() return hdr + def _ord(s): # in python2 it is char/str, while in py3 it is already int/bytes if isinstance(s, int): @@ -794,6 +835,7 @@ def _ord(s): else: return ord(s) + def __parse_packet_header(pgp_packet): """Parse pgp_packet header, return tag type and the rest of pgp_packet""" byte0 = _ord(pgp_packet[0]) @@ -828,6 +870,7 @@ def __parse_packet_header(pgp_packet): raise ValueError('Invalid OpenPGP packet length') return (tag, pgp_packet[offset:]) + def __subpacket_key_ids(subs): """Parse v4 signature subpackets and return a list of issuer key IDs""" res = [] @@ -847,6 +890,7 @@ def __subpacket_key_ids(subs): subs = subs[off + length:] return res + def get_sigpacket_key_id(sigpacket): """Return ID of the key used to create sigpacket as a hexadecimal string""" (tag, sigpacket) = __parse_packet_header(sigpacket) @@ -870,6 +914,7 @@ def get_sigpacket_key_id(sigpacket): 'Unknown PGP signature packet version %s' % _ord(sigpacket[0])) return hex_string(key_id) + def get_sighdr_key(sighdr): """Parse the sighdr and return the sigkey""" rh = RawHeader(sighdr) @@ -881,6 +926,7 @@ def get_sighdr_key(sighdr): else: return get_sigpacket_key_id(sig) + def splice_rpm_sighdr(sighdr, src, dst=None, bufsize=8192): """Write a copy of an rpm with signature header spliced in""" (start, size) = find_rpm_sighdr(src) @@ -901,6 +947,7 @@ def splice_rpm_sighdr(sighdr, src, dst=None, bufsize=8192): dst_fo.close() return dst + def get_rpm_header(f, ts=None): """Return the rpm header.""" if rpm is None: @@ -998,6 +1045,7 @@ def get_header_fields(X, fields, src_arch=False): ret[f] = get_header_field(hdr, f, src_arch=src_arch) return ret + def parse_NVR(nvr): """split N-V-R into dictionary of data""" ret = {} @@ -1018,6 +1066,7 @@ def parse_NVR(nvr): ret['name'] = ret['name'][epochIndex + 1:] return ret + def parse_NVRA(nvra): """split N-V-R.A.rpm into dictionary of data @@ -1061,6 +1110,7 @@ def check_NVR(nvr, strict=False): else: return False + def _check_NVR(nvr): if isinstance(nvr, six.string_types): nvr = parse_NVR(nvr) @@ -1107,6 +1157,7 @@ def is_debuginfo(name): return (name.endswith('-debuginfo') or name.endswith('-debugsource') or '-debuginfo-' in name) + def canonArch(arch): """Given an arch, return the "canonical" arch""" # XXX - this could stand to be smarter, and we should probably @@ -1132,6 +1183,7 @@ def canonArch(arch): else: return arch + def parse_arches(arches, to_list=False, strict=False, allow_none=False): """Normalize user input for a list of arches. @@ -1195,8 +1247,10 @@ class POMHandler(xml.sax.handler.ContentHandler): self.tag_content = None self.values.clear() + ENTITY_RE = re.compile(r'&[A-Za-z0-9]+;') + def parse_pom(path=None, contents=None): """ Parse the Maven .pom file return a map containing information @@ -1236,6 +1290,7 @@ def parse_pom(path=None, contents=None): raise GenericError('could not extract %s from POM: %s' % (field, (path or ''))) return values + def pom_to_maven_info(pominfo): """ Convert the output of parsing a POM into a format compatible @@ -1250,6 +1305,7 @@ def pom_to_maven_info(pominfo): 'version': pominfo['version']} return maveninfo + def maven_info_to_nvr(maveninfo): """ Convert the maveninfo to NVR-compatible format. @@ -1264,6 +1320,7 @@ def maven_info_to_nvr(maveninfo): nvr['package_name'] = nvr['name'] return nvr + def mavenLabel(maveninfo): """ Return a user-friendly label for the given maveninfo. maveninfo is @@ -1271,6 +1328,7 @@ def mavenLabel(maveninfo): """ return '%(group_id)s-%(artifact_id)s-%(version)s' % maveninfo + def hex_string(s): """Converts a string to a string of hex digits""" return ''.join(['%02x' % _ord(x) for x in s]) @@ -1339,6 +1397,7 @@ This is a meta-package that requires a defined group of packages """) return ''.join(data) + def generate_comps(groups, expand_groups=False): """Generate comps content from groups data""" def boolean_text(x): @@ -1635,11 +1694,14 @@ name=build return ''.join(parts) + def get_sequence_value(cursor, sequence): cursor.execute("""SELECT nextval(%(sequence)s)""", locals()) return cursor.fetchone()[0] # From Python Cookbook 2nd Edition, Recipe 8.6 + + def format_exc_plus(): """ Format the usual traceback information, followed by a listing of all the local variables in each frame. @@ -1669,6 +1731,7 @@ def format_exc_plus(): rv += "\n" return rv + def openRemoteFile(relpath, topurl=None, topdir=None, tempdir=None): """Open a file on the main server (read-only) @@ -2085,6 +2148,7 @@ class PathInfo(object): """Return the output directory for the task with the given id""" return self.work(volume=volume) + '/' + self.taskrelpath(task_id) + pathinfo = PathInfo() @@ -3171,6 +3235,7 @@ class DBHandler(logging.Handler): A handler class which writes logging records, appropriately formatted, to a database. """ + def __init__(self, cnx, table, mapping=None): """ Initialize the handler. @@ -3221,6 +3286,7 @@ class DBHandler(logging.Handler): except: self.handleError(record) + def formatTime(value): """Format a timestamp so it looks nicer""" if not value: @@ -3237,6 +3303,7 @@ def formatTime(value): else: return value + def formatTimeLong(value): """Format a timestamp to a more human-reable format, i.e.: Sat, 07 Sep 2002 00:00:01 GMT @@ -3248,6 +3315,7 @@ def formatTimeLong(value): localtime = time.mktime(time.strptime(formatTime(value), '%Y-%m-%d %H:%M:%S')) return time.strftime('%a, %d %b %Y %H:%M:%S %Z', time.localtime(localtime)) + def buildLabel(buildInfo, showEpoch=False): """Format buildInfo (dict) into a descriptive label.""" epoch = buildInfo.get('epoch') @@ -3262,6 +3330,7 @@ def buildLabel(buildInfo, showEpoch=False): buildInfo.get('version'), buildInfo.get('release')) + def _module_info(url): module_info = '' if '?' in url: @@ -3280,12 +3349,14 @@ def _module_info(url): else: return '%s:%s' % (repo_info, rev_info) + def taskLabel(taskInfo): try: return _taskLabel(taskInfo) except Exception: return "malformed task" + def _taskLabel(taskInfo): """Format taskInfo (dict) into a descriptive label.""" method = taskInfo['method'] @@ -3405,10 +3476,13 @@ def _taskLabel(taskInfo): else: return '%s (%s)' % (method, arch) + CONTROL_CHARS = [chr(i) for i in range(32)] NONPRINTABLE_CHARS = ''.join([c for c in CONTROL_CHARS if c not in '\r\n\t']) if six.PY3: NONPRINTABLE_CHARS_TABLE = dict.fromkeys(map(ord, NONPRINTABLE_CHARS), None) + + def removeNonprintable(value): # expects raw-encoded string, not unicode if six.PY2: @@ -3506,12 +3580,14 @@ def add_file_logger(logger, fn): handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s')) logging.getLogger(logger).addHandler(handler) + def add_stderr_logger(logger): handler = logging.StreamHandler() handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] {%(process)d} %(name)s:%(lineno)d %(message)s')) handler.setLevel(logging.DEBUG) logging.getLogger(logger).addHandler(handler) + def add_sys_logger(logger): # For remote logging; # address = ('host.example.com', logging.handlers.SysLogHandler.SYSLOG_UDP_PORT) @@ -3522,6 +3598,7 @@ def add_sys_logger(logger): handler.setLevel(logging.INFO) logging.getLogger(logger).addHandler(handler) + def add_mail_logger(logger, addr): """Adding e-mail logger @@ -3542,5 +3619,6 @@ def add_mail_logger(logger, addr): handler.setLevel(logging.ERROR) logging.getLogger(logger).addHandler(handler) + def remove_log_handler(logger, handler): logging.getLogger(logger).removeHandler(handler) diff --git a/koji/arch.py b/koji/arch.py index 25abff24..19c8fd98 100644 --- a/koji/arch.py +++ b/koji/arch.py @@ -97,6 +97,7 @@ _aux_vector = { "hwcap": 0, } + def legitMultiArchesInSameLib(arch=None): # this is completely crackrock - if anyone has a better way I # am all ears @@ -135,6 +136,8 @@ def canCoinstall(arch1, arch2): return False # this computes the difference between myarch and targetarch + + def archDifference(myarch, targetarch): if myarch == targetarch: return 1 @@ -145,9 +148,11 @@ def archDifference(myarch, targetarch): return 0 return 0 + def score(arch): return archDifference(canonArch, arch) + def isMultiLibArch(arch=None): """returns true if arch is a multilib arch, false if not""" if arch is None: @@ -164,6 +169,7 @@ def isMultiLibArch(arch=None): return 0 + def getBestArchFromList(archlist, myarch=None): """ return the best arch from the list for myarch if - myarch is not given, @@ -226,6 +232,7 @@ def getArchList(thisarch=None): archlist.append('noarch') return archlist + def _try_read_cpuinfo(): """ Try to read /proc/cpuinfo ... if we can't ignore errors (ie. proc not mounted). """ @@ -234,6 +241,7 @@ def _try_read_cpuinfo(): except: return [] + def _parse_auxv(): """ Read /proc/self/auxv and parse it into global dict for easier access later on, very similar to what rpm does. """ @@ -260,6 +268,7 @@ def _parse_auxv(): _aux_vector["hwcap"] = at_val offset = offset + fmtlen + def getCanonX86Arch(arch): # if arch == "i586": @@ -285,6 +294,7 @@ def getCanonX86Arch(arch): return arch + def getCanonARMArch(arch): # the %{_target_arch} macro in rpm will let us know the abi we are using target = rpm.expandMacro('%{_target_cpu}') @@ -294,6 +304,7 @@ def getCanonARMArch(arch): return target return arch + def getCanonPPCArch(arch): # FIXME: should I do better handling for mac, etc? if arch != "ppc64": @@ -324,6 +335,7 @@ def getCanonPPCArch(arch): return "ppc64iseries" return arch + def getCanonSPARCArch(arch): # Deal with sun4v, sun4u, sun4m cases SPARCtype = None @@ -348,6 +360,7 @@ def getCanonSPARCArch(arch): return "sparcv8" return arch + def getCanonX86_64Arch(arch): if arch != "x86_64": return arch @@ -366,6 +379,7 @@ def getCanonX86_64Arch(arch): return "ia32e" return arch + def getCanonArch(skipRpmPlatform=0): if not skipRpmPlatform and os.access("/etc/rpm/platform", os.R_OK): try: @@ -395,9 +409,12 @@ def getCanonArch(skipRpmPlatform=0): return arch + canonArch = getCanonArch() # this gets you the "compat" arch of a biarch pair + + def getMultiArchInfo(arch=canonArch): if arch in multilibArches: return multilibArches[arch] @@ -408,6 +425,8 @@ def getMultiArchInfo(arch=canonArch): # get the best usual userspace arch for the arch we're on. this is # our arch unless we're on an arch that uses the secondary as its # userspace (eg ppc64, sparc64) + + def getBestArch(myarch=None): if myarch: arch = myarch @@ -422,6 +441,7 @@ def getBestArch(myarch=None): return arch + def getBaseArch(myarch=None): """returns 'base' arch for myarch, if specified, or canonArch if not. base arch is the arch before noarch in the arches dict if myarch is not @@ -467,6 +487,7 @@ def getBaseArch(myarch=None): class ArchStorage(object): """class for keeping track of what arch we have set and doing various permutations based on it""" + def __init__(self): self.canonarch = None self.basearch = None diff --git a/koji/auth.py b/koji/auth.py index 5a038544..124c1b33 100644 --- a/koji/auth.py +++ b/koji/auth.py @@ -546,7 +546,6 @@ class Session(object): c.execute(q, {}) (session_id,) = c.fetchone() - # add session id to database q = """ INSERT INTO sessions (id, user_id, key, hostip, authtype, master) @@ -799,6 +798,7 @@ def get_user_groups(user_id): c.execute(q, locals()) return dict(c.fetchall()) + def get_user_perms(user_id): c = context.cnx.cursor() q = """SELECT name @@ -808,6 +808,7 @@ def get_user_perms(user_id): # return a list of permissions by name return [row[0] for row in c.fetchall()] + def get_user_data(user_id): c = context.cnx.cursor() fields = ('name', 'status', 'usertype') @@ -818,28 +819,36 @@ def get_user_data(user_id): return None return dict(zip(fields, row)) + def login(*args, **opts): return context.session.login(*args, **opts) + def krbLogin(*args, **opts): return context.session.krbLogin(*args, **opts) + def sslLogin(*args, **opts): return context.session.sslLogin(*args, **opts) + def logout(): return context.session.logout() + def subsession(): return context.session.subsession() + def logoutChild(session_id): return context.session.logoutChild(session_id) + def exclusiveSession(*args, **opts): """Make this session exclusive""" return context.session.makeExclusive(*args, **opts) + def sharedSession(): """Drop out of exclusive mode""" return context.session.makeShared() diff --git a/koji/context.py b/koji/context.py index 7ec2624a..5d758716 100644 --- a/koji/context.py +++ b/koji/context.py @@ -32,6 +32,7 @@ import six.moves._thread class _data(object): pass + class ThreadLocal(object): def __init__(self): object.__setattr__(self, '_tdict', {}) diff --git a/koji/daemon.py b/koji/daemon.py index d9cfa7e8..892182bf 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -84,6 +84,7 @@ def incremental_upload(session, fname, fd, path, retries=5, logger=None): logger.error("Error uploading file %s to %s at offset %d" % (fname, path, offset)) break + def fast_incremental_upload(session, fname, fd, path, retries, logger): """Like incremental_upload, but use the fast upload mechanism""" @@ -108,6 +109,7 @@ def fast_incremental_upload(session, fname, fd, path, retries, logger): logger.error("Error uploading file %s to %s at offset %d" % (fname, path, offset)) break + def log_output(session, path, args, outfile, uploadpath, cwd=None, logerror=0, append=0, chroot=None, env=None): """Run command with output redirected. If chroot is not None, chroot to the directory specified before running the command.""" @@ -400,6 +402,7 @@ class SCM(object): update_checkout_cmd = None update_checkout_dir = None env = None + def _run(cmd, chdir=None, fatal=False, log=True, _count=[0]): if globals().get('KOJIKAMID'): # we've been inserted into kojikamid, use its run() diff --git a/koji/db.py b/koji/db.py index 3d36b876..7a012488 100644 --- a/koji/db.py +++ b/koji/db.py @@ -55,6 +55,7 @@ _DBopts = None # but play it safe anyway. _DBconn = context.ThreadLocal() + class DBWrapper: def __init__(self, cnx): self.cnx = cnx @@ -151,13 +152,16 @@ def provideDBopts(**opts): if _DBopts is None: _DBopts = dict([i for i in opts.items() if i[1] is not None]) + def setDBopts(**opts): global _DBopts _DBopts = opts + def getDBopts(): return _DBopts + def connect(): logger = logging.getLogger('koji.db') global _DBconn diff --git a/koji/plugin.py b/koji/plugin.py index 570599c0..012a80f0 100644 --- a/koji/plugin.py +++ b/koji/plugin.py @@ -58,6 +58,7 @@ callbacks = { 'postSCMCheckout': [], } + class PluginTracker(object): def __init__(self, path=None, prefix='_koji_plugin__'): @@ -113,6 +114,7 @@ def export(f): setattr(f, 'exported', True) return f + def export_cli(f): """a decorator that marks a function as exported for CLI @@ -122,6 +124,7 @@ def export_cli(f): setattr(f, 'exported_cli', True) return f + def export_as(alias): """returns a decorator that marks a function as exported and gives it an alias @@ -133,6 +136,7 @@ def export_as(alias): return f return dec + def export_in(module, alias=None): """returns a decorator that marks a function as exported with a module prepended @@ -150,6 +154,7 @@ def export_in(module, alias=None): return f return dec + def callback(*cbtypes): """A decorator that indicates a function is a callback. cbtypes is a list of callback types to register for. Valid @@ -162,6 +167,7 @@ def callback(*cbtypes): return f return dec + def ignore_error(f): """a decorator that marks a callback as ok to fail diff --git a/koji/policy.py b/koji/policy.py index d29ee510..729e02e5 100644 --- a/koji/policy.py +++ b/koji/policy.py @@ -50,12 +50,14 @@ class BaseSimpleTest(object): class TrueTest(BaseSimpleTest): name = 'true' + def run(self, data): return True class FalseTest(BaseSimpleTest): name = 'false' + def run(self, data): return False @@ -97,6 +99,7 @@ class BoolTest(BaseSimpleTest): """ name = 'bool' field = None + def run(self, data): args = self.str.split()[1:] if self.field is None: @@ -121,6 +124,7 @@ class MatchTest(BaseSimpleTest): """ name = 'match' field = None + def run(self, data): args = self.str.split()[1:] if self.field is None: diff --git a/koji/rpmdiff.py b/koji/rpmdiff.py index 430b9ced..8dcc1791 100644 --- a/koji/rpmdiff.py +++ b/koji/rpmdiff.py @@ -36,6 +36,7 @@ class BytesJSONEncoder(json.JSONEncoder): return o.decode('utf-8') return json.JSONEncoder.default(self, o) + class Rpmdiff: # constants diff --git a/koji/server.py b/koji/server.py index 78284aa6..06291d52 100644 --- a/koji/server.py +++ b/koji/server.py @@ -19,8 +19,10 @@ # Authors: # Mike McLean + class ServerError(Exception): """Base class for our server-side-only exceptions""" + class ServerRedirect(ServerError): """Used to handle redirects""" diff --git a/koji/tasks.py b/koji/tasks.py index 00e845c7..ddc2153c 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -55,6 +55,7 @@ def scan_mounts(topdir): mplist.sort(reverse=True) return mplist + def umount_all(topdir): "Unmount every mount under topdir" logger = logging.getLogger("koji.build") @@ -69,6 +70,7 @@ def umount_all(topdir): if remain: raise koji.GenericError("Unmounting incomplete: %r" % remain) + def safe_rmtree(path, unmount=False, strict=True): logger = logging.getLogger("koji.build") if unmount: @@ -103,6 +105,7 @@ class ServerExit(Exception): """Raised to shutdown the server""" pass + class ServerRestart(Exception): """Raised to restart the server""" pass @@ -431,7 +434,6 @@ class BaseTaskHandler(object): return dict(self.session.host.taskWaitResults(self.id, finished, canfail=canfail)) - def getUploadDir(self): return koji.pathinfo.taskrelpath(self.id) @@ -579,7 +581,6 @@ class BaseTaskHandler(object): repo_info = self.wait(task_id)[task_id] return repo_info - def run_callbacks(self, plugin, *args, **kwargs): if 'taskinfo' not in kwargs: try: @@ -595,6 +596,7 @@ class BaseTaskHandler(object): class FakeTask(BaseTaskHandler): Methods = ['someMethod'] Foreground = True + def handler(self, *args): self.logger.info("This is a fake task. Args: " + str(args)) return 42 @@ -603,17 +605,21 @@ class FakeTask(BaseTaskHandler): class SleepTask(BaseTaskHandler): Methods = ['sleep'] _taskWeight = 0.25 + def handler(self, n): self.logger.info("Sleeping for %s seconds" % n) time.sleep(n) self.logger.info("Finished sleeping") + class ForkTask(BaseTaskHandler): Methods = ['fork'] + def handler(self, n=5, m=37): for i in range(n): os.spawnvp(os.P_NOWAIT, 'sleep', ['sleep', str(m)]) + class WaitTestTask(BaseTaskHandler): """ Tests self.wait() @@ -624,6 +630,7 @@ class WaitTestTask(BaseTaskHandler): """ Methods = ['waittest'] _taskWeight = 0.1 + def handler(self, count, seconds=10): tasks = [] for i in range(count): @@ -638,6 +645,7 @@ class WaitTestTask(BaseTaskHandler): class SubtaskTask(BaseTaskHandler): Methods = ['subtask'] _taskWeight = 0.1 + def handler(self, n=4): if n > 0: task_id = self.session.host.subtask(method='subtask', @@ -657,6 +665,7 @@ class DefaultTask(BaseTaskHandler): """Used when no matching method is found""" Methods = ['default'] _taskWeight = 0.1 + def handler(self, *args, **opts): raise koji.GenericError("Invalid method: %s" % self.method) @@ -665,6 +674,7 @@ class ShutdownTask(BaseTaskHandler): Methods = ['shutdown'] _taskWeight = 0.0 Foreground = True + def handler(self): # note: this is a foreground task raise ServerExit @@ -676,6 +686,7 @@ class RestartTask(BaseTaskHandler): Methods = ['restart'] _taskWeight = 0.1 Foreground = True + def handler(self, host): # note: this is a foreground task if host['id'] != self.session.host.getID(): @@ -690,6 +701,7 @@ class RestartVerifyTask(BaseTaskHandler): Methods = ['restartVerify'] _taskWeight = 0.1 Foreground = True + def handler(self, task_id, host): # note: this is a foreground task tinfo = self.session.getTaskInfo(task_id) @@ -708,6 +720,7 @@ class RestartHostsTask(BaseTaskHandler): Methods = ['restartHosts'] _taskWeight = 0.1 + def handler(self, options=None): if options is None: options = {} @@ -784,6 +797,7 @@ class DependantTask(BaseTaskHandler): if subtasks: self.wait(subtasks, all=True) + class MultiPlatformTask(BaseTaskHandler): def buildWrapperRPM(self, spec_url, build_task_id, build_target, build, repo_id, **opts): task = self.session.getTaskInfo(build_task_id) diff --git a/koji/util.py b/koji/util.py index b99b0fe5..919f5929 100644 --- a/koji/util.py +++ b/koji/util.py @@ -52,6 +52,7 @@ def deprecated(message): warnings.simplefilter('always', DeprecationWarning) warnings.warn(message, DeprecationWarning) + def _changelogDate(cldate): return time.strftime('%a %b %d %Y', time.strptime(koji.formatTime(cldate), '%Y-%m-%d %H:%M:%S')) @@ -69,6 +70,7 @@ def formatChangelog(entries): koji._fix_print(entry['text'])) return result + DATE_RE = re.compile(r'(\d+)-(\d+)-(\d+)') TIME_RE = re.compile(r'(\d+):(\d+):(\d+)') @@ -498,7 +500,6 @@ def move_and_symlink(src, dst, relative=True, create_dir=False): os.symlink(dst, src) - def joinpath(path, *paths): """A wrapper around os.path.join that limits directory traversal""" diff --git a/koji/xmlrpcplus.py b/koji/xmlrpcplus.py index 9cb9b9c9..0ca27930 100644 --- a/koji/xmlrpcplus.py +++ b/koji/xmlrpcplus.py @@ -55,7 +55,6 @@ if six.PY2: ExtendedMarshaller.dispatch[long] = ExtendedMarshaller.dump_int # noqa: F821 - def dumps(params, methodname=None, methodresponse=None, encoding=None, allow_none=1, marshaller=None): """encode an xmlrpc request or response diff --git a/plugins/hub/protonmsg.py b/plugins/hub/protonmsg.py index 3f164d7e..2c1874aa 100644 --- a/plugins/hub/protonmsg.py +++ b/plugins/hub/protonmsg.py @@ -22,6 +22,7 @@ from koji.plugin import callback, convert_datetime, ignore_error CONFIG_FILE = '/etc/koji-hub/plugins/protonmsg.conf' CONFIG = None + class TimeoutHandler(MessagingHandler): def __init__(self, url, msgs, conf, *args, **kws): super(TimeoutHandler, self).__init__(*args, **kws) @@ -151,6 +152,7 @@ def queue_msg(address, props, data): body = json.dumps(data, default=json_serialize) msgs.append((address, props, body)) + @convert_datetime @callback('postPackageListChange') def prep_package_list_change(cbtype, *args, **kws): @@ -162,6 +164,7 @@ def prep_package_list_change(cbtype, *args, **kws): 'user': kws['user']['name']} queue_msg(address, props, kws) + @convert_datetime @callback('postTaskStateChange') def prep_task_state_change(cbtype, *args, **kws): @@ -177,6 +180,7 @@ def prep_task_state_change(cbtype, *args, **kws): 'new': kws['new']} queue_msg(address, props, kws) + @convert_datetime @callback('postBuildStateChange') def prep_build_state_change(cbtype, *args, **kws): @@ -196,6 +200,7 @@ def prep_build_state_change(cbtype, *args, **kws): 'new': new} queue_msg(address, props, kws) + @convert_datetime @callback('postImport') def prep_import(cbtype, *args, **kws): @@ -207,6 +212,7 @@ def prep_import(cbtype, *args, **kws): 'release': kws['build']['release']} queue_msg(address, props, kws) + @convert_datetime @callback('postRPMSign') def prep_rpm_sign(cbtype, *args, **kws): @@ -224,6 +230,7 @@ def prep_rpm_sign(cbtype, *args, **kws): 'rpm_arch': kws['rpm']['arch']} queue_msg(address, props, kws) + def _prep_tag_msg(address, cbtype, kws): build = kws['build'] props = {'type': cbtype[4:], @@ -234,16 +241,19 @@ def _prep_tag_msg(address, cbtype, kws): 'user': kws['user']['name']} queue_msg(address, props, kws) + @convert_datetime @callback('postTag') def prep_tag(cbtype, *args, **kws): _prep_tag_msg('build.tag', cbtype, kws) + @convert_datetime @callback('postUntag') def prep_untag(cbtype, *args, **kws): _prep_tag_msg('build.untag', cbtype, kws) + @convert_datetime @callback('postRepoInit') def prep_repo_init(cbtype, *args, **kws): @@ -253,6 +263,7 @@ def prep_repo_init(cbtype, *args, **kws): 'repo_id': kws['repo_id']} queue_msg(address, props, kws) + @convert_datetime @callback('postRepoDone') def prep_repo_done(cbtype, *args, **kws): @@ -263,6 +274,7 @@ def prep_repo_done(cbtype, *args, **kws): 'expire': kws['expire']} queue_msg(address, props, kws) + @ignore_error @convert_datetime @callback('postCommit') diff --git a/plugins/hub/runroot_hub.py b/plugins/hub/runroot_hub.py index a2b52698..21e95840 100644 --- a/plugins/hub/runroot_hub.py +++ b/plugins/hub/runroot_hub.py @@ -27,6 +27,7 @@ def get_channel_arches(channel): ret[koji.canonArch(a)] = 1 return ret + @export def runroot(tagInfo, arch, command, channel=None, **opts): """ Create a runroot task """ diff --git a/setup.py b/setup.py index bfd8bf7b..6f800fa9 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ def get_install_requires(): return requires + setup( name="koji", version="1.20.0", diff --git a/util/koji-gc b/util/koji-gc index 66c7bc15..f28b2f4d 100755 --- a/util/koji-gc +++ b/util/koji-gc @@ -40,6 +40,7 @@ def _(args): """Stub function for translation""" return args + def get_options(): """process options from command line and config file""" @@ -238,6 +239,7 @@ def get_options(): return options, args + def check_tag(name): """Check tag name against options and determine if we should process it @@ -258,6 +260,7 @@ def check_tag(name): # not ignored and no filter specified return True + def check_package(name): """Check package name against options and determine if we should process it @@ -273,6 +276,7 @@ def check_package(name): # no filter specified return True + time_units = { 'second': 1, 'minute': 60, @@ -288,11 +292,14 @@ time_unit_aliases = [ ['minute', 'minutes', 'min', 'mins'], ['second', 'seconds', 'sec', 'secs', 's'], ] + + def parse_duration(str): """Parse time duration from string, returns duration in seconds""" ret = 0 n = None unit = None + def parse_num(s): try: return int(s) @@ -338,16 +345,19 @@ def parse_duration(str): unit = None return ret + def error(msg=None, code=1): if msg: sys.stderr.write(msg + "\n") sys.stderr.flush() sys.exit(code) + def warn(msg): sys.stderr.write(msg + "\n") sys.stderr.flush() + def ensure_connection(session): try: ret = session.getAPIVersion() @@ -356,6 +366,7 @@ def ensure_connection(session): if ret != koji.API_VERSION: warn(_("WARNING: The server is at API version %d and the client is at %d" % (ret, koji.API_VERSION))) + def has_krb_creds(): if krbV is None: return False @@ -367,6 +378,7 @@ def has_krb_creds(): except krbV.Krb5Error: return False + def activate_session(session): """Test and login the session is applicable""" global options @@ -395,6 +407,7 @@ def activate_session(session): if options.debug: print("successfully connected to hub") + def send_warning_notice(owner_name, builds): if not options.mail: return @@ -587,6 +600,7 @@ def handle_trash(): # run all packageListAdd/tagBuildBypass finally mcall.call_all() + def protected_sig(keys): """Check list of keys and see if any are protected @@ -612,6 +626,7 @@ def handle_salvage(): run this action only.""" return handle_delete(just_salvage=True) + def salvage_build(binfo): """Removes trashcan tag from a build and prints a message""" if options.test: @@ -621,6 +636,7 @@ def salvage_build(binfo): print("Untagging from trashcan: %(nvr)s" % binfo) session.untagBuildBypass(options.trashcan_tag, binfo['id'], force=True) + def handle_delete(just_salvage=False): """Delete builds that have been in the trashcan for long enough @@ -793,6 +809,7 @@ def read_policies(fn=None): fo.close() return ret + def scan_policies(str): """Read tag gc policies from a string @@ -802,8 +819,10 @@ def scan_policies(str): tests = koji.policy.findSimpleTests(globals()) return koji.policy.SimpleRuleSet(str.splitlines(), tests) + build_sig_cache = {} + def get_build_sigs(build, cache=False): if cache and build in build_sig_cache: return build_sig_cache[build] @@ -825,6 +844,7 @@ def get_build_sigs(build, cache=False): ret = build_sig_cache[build] = to_list(keys.keys()) return ret + def handle_prune(): """Untag old builds according to policy @@ -966,6 +986,7 @@ def handle_prune(): # server issue pass + if __name__ == "__main__": options, args = get_options() diff --git a/util/koji-shadow b/util/koji-shadow index 2b770f58..3377b4da 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -60,16 +60,19 @@ def _(args): """Stub function for translation""" return args + def log(str): global logfile print("%s" % str) if logfile is not None: os.write(logfile, "%s\n" % str) + class SubOption(object): """A simple container to help with tracking ConfigParser data""" pass + def get_options(): """process options from command line and config file""" @@ -189,6 +192,7 @@ def get_options(): return options, args + time_units = { 'second': 1, 'minute': 60, @@ -204,11 +208,14 @@ time_unit_aliases = [ ['minute', 'minutes', 'min', 'mins'], ['second', 'seconds', 'sec', 'secs', 's'], ] + + def parse_duration(str): """Parse time duration from string, returns duration in seconds""" ret = 0 n = None unit = None + def parse_num(s): try: return int(s) @@ -254,16 +261,19 @@ def parse_duration(str): unit = None return ret + def error(msg=None, code=1): if msg: sys.stderr.write(msg + "\n") sys.stderr.flush() sys.exit(code) + def warn(msg): sys.stderr.write(msg + "\n") sys.stderr.flush() + def ensure_connection(session): try: ret = session.getAPIVersion() @@ -273,6 +283,7 @@ def ensure_connection(session): warn(_("WARNING: The server is at API version %d and the client is at " "%d" % (ret, koji.API_VERSION))) + def activate_session(session): """Test and login the session is applicable""" global options @@ -307,6 +318,7 @@ def activate_session(session): if options.debug: log("successfully connected to hub") + def _unique_path(prefix): """Create a unique path fragment by appending a path component to prefix. The path component will consist of a string of letter and numbers diff --git a/util/koji-sweep-db b/util/koji-sweep-db index b1896bc3..e52a02fd 100755 --- a/util/koji-sweep-db +++ b/util/koji-sweep-db @@ -193,7 +193,6 @@ if __name__ == "__main__": if opts['DBHost'] is None: opts['DBHost'] = opts['DBhost'] - koji.db.provideDBopts(database=opts["DBName"], user=opts["DBUser"], password=opts.get("DBPass", None), diff --git a/util/kojira b/util/kojira index 44e3404e..942ffe6d 100755 --- a/util/kojira +++ b/util/kojira @@ -43,6 +43,7 @@ from koji.util import parseStatus, rmtree, to_list tag_cache = {} + def getTag(session, tag, event=None): """A caching version of the hub call""" cache = tag_cache @@ -724,7 +725,6 @@ class RepoManager(object): self.logger.info('Needed tags count went from %i to %i', n_need, len(self.needed_tags)) - def regenRepos(self): """Trigger newRepo tasks for needed tags""" @@ -822,6 +822,7 @@ def start_regen_loop(session, repomgr): def main(options, session): repomgr = RepoManager(options, session) repomgr.readCurrentRepos() + def shutdown(*args): raise SystemExit signal.signal(signal.SIGTERM, shutdown) @@ -864,6 +865,7 @@ def main(options, session): finally: session.logout() + def get_options(): """process options from command line and config file""" # parse command line args @@ -976,6 +978,7 @@ def get_options(): setattr(options, name, fn) return options + def quit(msg=None, code=1): if msg: logging.getLogger("koji.repo").error(msg) @@ -983,6 +986,7 @@ def quit(msg=None, code=1): sys.stderr.flush() sys.exit(code) + if __name__ == "__main__": options = get_options() diff --git a/vm/kojikamid.py b/vm/kojikamid.py index d5d88fc4..8f231a09 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -56,14 +56,17 @@ KOJIKAMID = True # INSERT kojikamid dup # + class fakemodule(object): pass + # make parts of the above insert accessible as koji.X koji = fakemodule() koji.GenericError = GenericError # noqa: F821 koji.BuildError = BuildError # noqa: F821 + def encode_int(n): """If n is too large for a 32bit signed, convert it to a string""" if n <= 2147483647: @@ -71,6 +74,7 @@ def encode_int(n): # else return str(n) + class WindowsBuild(object): LEADING_CHAR = re.compile('^[^A-Za-z_]') @@ -529,6 +533,7 @@ class WindowsBuild(object): self.expireBuildroot() return self.gatherResults() + def run(cmd, chdir=None, fatal=False, log=True): global logfd output = '' @@ -558,6 +563,7 @@ def run(cmd, chdir=None, fatal=False, log=True): raise BuildError(msg) # noqa: F821 return ret, output + def find_net_info(): """ Find the network gateway configured for this VM. @@ -586,6 +592,7 @@ def find_net_info(): gateway = None return macaddr, gateway + def upload_file(server, prefix, path): """upload a single file to the vmd""" logger = logging.getLogger('koji.vm') @@ -606,6 +613,7 @@ def upload_file(server, prefix, path): server.verifyChecksum(path, digest, 'md5') logger.info('Uploaded %s (%s bytes, md5: %s)', destpath, offset, digest) + def get_mgmt_server(): """Get a ServerProxy object we can use to retrieve task info""" logger = logging.getLogger('koji.vm') @@ -624,6 +632,7 @@ def get_mgmt_server(): logger.debug('found task-specific port %s', task_port) return six.moves.xmlrpc_client.ServerProxy('http://%s:%s/' % (gateway, task_port), allow_none=True) + def get_options(): """handle usage and parse options""" usage = """%prog [options] @@ -637,6 +646,7 @@ def get_options(): (options, args) = parser.parse_args() return options + def setup_logging(opts): global logfile, logfd logger = logging.getLogger('koji.vm') @@ -651,11 +661,13 @@ def setup_logging(opts): logger.addHandler(handler) return handler + def log_local(msg): tb = ''.join(traceback.format_exception(*sys.exc_info())) sys.stderr.write('%s: %s\n' % (time.ctime(), msg)) sys.stderr.write(tb) + def stream_logs(server, handler, builds): """Stream logs incrementally to the server. The global logfile will always be streamed. @@ -693,6 +705,7 @@ def stream_logs(server, handler, builds): log_local('error uploading %s' % relpath) time.sleep(1) + def fail(server, handler): """do the right thing when a build fails""" global logfile, logfd @@ -719,6 +732,7 @@ def fail(server, handler): logfile = '/tmp/build.log' logfd = None + def main(): prog = os.path.basename(sys.argv[0]) opts = get_options() diff --git a/vm/kojivmd b/vm/kojivmd index 201ca2dd..82dacf6c 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -68,6 +68,8 @@ def libvirt_callback(ignore, err): if err[3] != libvirt.VIR_ERR_ERROR: # Don't log libvirt errors: global error handler will do that logging.warn("Non-error from libvirt: '%s'", err[2]) + + libvirt.registerErrorHandler(f=libvirt_callback, ctx=None) @@ -191,6 +193,7 @@ def get_options(): return options + def quit(msg=None, code=1): if msg: logging.getLogger("koji.vm").error(msg) @@ -198,6 +201,7 @@ def quit(msg=None, code=1): sys.stderr.flush() sys.exit(code) + def main(options, session): logger = logging.getLogger("koji.vm") logger.info('Starting up') @@ -209,8 +213,10 @@ def main(options, session): for name in options.plugin: logger.info('Loading plugin: %s', name) tm.scanPlugin(pt.load(name)) + def shutdown(*args): raise SystemExit + def restart(*args): logger.warn("Initiating graceful restart") tm.restart_pending = True @@ -416,6 +422,7 @@ class WinBuildTask(MultiPlatformTask): parent=self.id) self.wait(tag_task_id) + class VMExecTask(BaseTaskHandler): """ Handles the startup, state-tracking, and shutdown of a VM @@ -883,6 +890,7 @@ class VMExecTask(BaseTaskHandler): else: raise koji.BuildError(self.output) + class VMTaskManager(TaskManager): def __init__(self, options, session): super(VMTaskManager, self).__init__(options, session) diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index d1b49108..dd128a39 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -48,6 +48,7 @@ _sortbyname = lambda x: x['name'] # loggers authlogger = logging.getLogger('koji.auth') + def _setUserCookie(environ, user): options = environ['koji.options'] # include the current time in the cookie so we can verify that @@ -72,6 +73,7 @@ def _setUserCookie(environ, user): environ['koji.headers'].append(['Set-Cookie', out]) environ['koji.headers'].append(['Cache-Control', 'no-cache="set-cookie"']) + def _clearUserCookie(environ): cookies = six.moves.http_cookies.SimpleCookie() cookies['user'] = '' @@ -81,6 +83,7 @@ def _clearUserCookie(environ): out = c.OutputString() environ['koji.headers'].append(['Set-Cookie', out]) + def _getUserCookie(environ): options = environ['koji.options'] cookies = six.moves.http_cookies.SimpleCookie(environ.get('HTTP_COOKIE', '')) @@ -118,6 +121,7 @@ def _getUserCookie(environ): # Otherwise, cookie is valid and current return user + def _krbLogin(environ, session, principal): options = environ['koji.options'] wprinc = options['WebPrincipal'] @@ -126,6 +130,7 @@ def _krbLogin(environ, session, principal): return session.krb_login(principal=wprinc, keytab=keytab, ccache=ccache, proxyuser=principal) + def _sslLogin(environ, session, username): options = environ['koji.options'] client_cert = options['WebCert'] @@ -134,6 +139,7 @@ def _sslLogin(environ, session, username): return session.ssl_login(client_cert, None, server_ca, proxyuser=username) + def _assertLogin(environ): session = environ['koji.session'] options = environ['koji.options'] @@ -165,6 +171,7 @@ def _assertLogin(environ): _redirect(environ, 'login') assert False # pragma: no cover + def _getServer(environ): opts = environ['koji.options'] s_opts = {'krbservice': opts['KrbService'], @@ -186,6 +193,7 @@ def _getServer(environ): environ['koji.session'] = session return session + def _construct_url(environ, page): port = environ['SERVER_PORT'] host = environ['SERVER_NAME'] @@ -195,14 +203,17 @@ def _construct_url(environ, page): return "%s://%s%s" % (url_scheme, host, page) return "%s://%s:%s%s" % (url_scheme, host, port, page) + def _getBaseURL(environ): base = environ['SCRIPT_NAME'] return _construct_url(environ, base) + def _redirect(environ, location): environ['koji.redirect'] = location raise ServerRedirect + def _redirectBack(environ, page, forceSSL): if page: # We'll work with the page we were given @@ -227,6 +238,7 @@ def _redirectBack(environ, page, forceSSL): # and redirect to the page _redirect(environ, page) + def login(environ, page=None): session = _getServer(environ) options = environ['koji.options'] @@ -270,6 +282,7 @@ def login(environ, page=None): # To protect the session cookie, we must forceSSL _redirectBack(environ, page, forceSSL=True) + def logout(environ, page=None): user = _getUserCookie(environ) _clearUserCookie(environ) @@ -278,6 +291,7 @@ def logout(environ, page=None): _redirectBack(environ, page, forceSSL=False) + def index(environ, packageOrder='package_name', packageStart=None): values = _initValues(environ) server = _getServer(environ) @@ -326,6 +340,7 @@ def index(environ, packageOrder='package_name', packageStart=None): return _genHTML(environ, 'index.chtml') + def notificationedit(environ, notificationID): server = _getServer(environ) _assertLogin(environ) @@ -371,6 +386,7 @@ def notificationedit(environ, notificationID): return _genHTML(environ, 'notificationedit.chtml') + def notificationcreate(environ): server = _getServer(environ) _assertLogin(environ) @@ -415,6 +431,7 @@ def notificationcreate(environ): return _genHTML(environ, 'notificationedit.chtml') + def notificationdelete(environ, notificationID): server = _getServer(environ) _assertLogin(environ) @@ -428,6 +445,7 @@ def notificationdelete(environ, notificationID): _redirect(environ, 'index') + # All Tasks _TASKS = ['build', 'buildSRPMFromSCM', @@ -463,6 +481,7 @@ _TOPLEVEL_TASKS = ['build', 'buildNotification', 'chainbuild', 'maven', 'chainma # Tasks that can have children _PARENT_TASKS = ['build', 'chainbuild', 'maven', 'chainmaven', 'winbuild', 'newRepo', 'distRepo', 'wrapperRPM', 'livecd', 'appliance', 'image', 'livemedia'] + def tasks(environ, owner=None, state='active', view='tree', method='all', hostID=None, channelID=None, start=None, order='-id'): values = _initValues(environ, 'Tasks', 'tasks') server = _getServer(environ) @@ -566,6 +585,7 @@ def tasks(environ, owner=None, state='active', view='tree', method='all', hostID return _genHTML(environ, 'tasks.chtml') + def taskinfo(environ, taskID): server = _getServer(environ) values = _initValues(environ, 'Task Info', 'tasks') @@ -712,6 +732,7 @@ def taskinfo(environ, taskID): values['params_parsed'] = None return _genHTML(environ, 'taskinfo.chtml') + def taskstatus(environ, taskID): server = _getServer(environ) @@ -726,6 +747,7 @@ def taskstatus(environ, taskID): output += '%s:%s:%s\n' % (volume, filename, file_stats['st_size']) return output + def resubmittask(environ, taskID): server = _getServer(environ) _assertLogin(environ) @@ -734,6 +756,7 @@ def resubmittask(environ, taskID): newTaskID = server.resubmitTask(taskID) _redirect(environ, 'taskinfo?taskID=%i' % newTaskID) + def canceltask(environ, taskID): server = _getServer(environ) _assertLogin(environ) @@ -742,11 +765,13 @@ def canceltask(environ, taskID): server.cancelTask(taskID) _redirect(environ, 'taskinfo?taskID=%i' % taskID) + def _sortByExtAndName(item): """Sort filename tuples key function, first by extension, and then by name.""" kRoot, kExt = os.path.splitext(os.path.basename(item[1])) return (kExt, kRoot) + def getfile(environ, taskID, name, volume='DEFAULT', offset=None, size=None): server = _getServer(environ) taskID = int(taskID) @@ -810,6 +835,7 @@ def _chunk_file(server, environ, taskID, name, offset, size, volume): offset += content_length remaining -= content_length + def tags(environ, start=None, order=None, childID=None): values = _initValues(environ, 'Tags', 'tags') server = _getServer(environ) @@ -830,8 +856,10 @@ def tags(environ, start=None, order=None, childID=None): return _genHTML(environ, 'tags.chtml') + _PREFIX_CHARS = [chr(char) for char in list(range(48, 58)) + list(range(97, 123))] + def packages(environ, tagID=None, userID=None, order='package_name', start=None, prefix=None, inherited='1'): values = _initValues(environ, 'Packages', 'packages') server = _getServer(environ) @@ -866,6 +894,7 @@ def packages(environ, tagID=None, userID=None, order='package_name', start=None, return _genHTML(environ, 'packages.chtml') + def packageinfo(environ, packageID, tagOrder='name', tagStart=None, buildOrder='-completion_time', buildStart=None): values = _initValues(environ, 'Package Info', 'packages') server = _getServer(environ) @@ -888,6 +917,7 @@ def packageinfo(environ, packageID, tagOrder='name', tagStart=None, buildOrder=' return _genHTML(environ, 'packageinfo.chtml') + def taginfo(environ, tagID, all='0', packageOrder='package_name', packageStart=None, buildOrder='-completion_time', buildStart=None, childID=None): values = _initValues(environ, 'Tag Info', 'tags') server = _getServer(environ) @@ -943,6 +973,7 @@ def taginfo(environ, tagID, all='0', packageOrder='package_name', packageStart=N return _genHTML(environ, 'taginfo.chtml') + def tagcreate(environ): server = _getServer(environ) _assertLogin(environ) @@ -978,6 +1009,7 @@ def tagcreate(environ): return _genHTML(environ, 'tagedit.chtml') + def tagedit(environ, tagID): server = _getServer(environ) _assertLogin(environ) @@ -1020,6 +1052,7 @@ def tagedit(environ, tagID): return _genHTML(environ, 'tagedit.chtml') + def tagdelete(environ, tagID): server = _getServer(environ) _assertLogin(environ) @@ -1033,6 +1066,7 @@ def tagdelete(environ, tagID): _redirect(environ, 'tags') + def tagparent(environ, tagID, parentID, action): server = _getServer(environ) _assertLogin(environ) @@ -1096,6 +1130,7 @@ def tagparent(environ, tagID, parentID, action): _redirect(environ, 'taginfo?tagID=%i' % tag['id']) + def externalrepoinfo(environ, extrepoID): values = _initValues(environ, 'External Repo Info', 'tags') server = _getServer(environ) @@ -1111,6 +1146,7 @@ def externalrepoinfo(environ, extrepoID): return _genHTML(environ, 'externalrepoinfo.chtml') + def buildinfo(environ, buildID): values = _initValues(environ, 'Build Info', 'builds') server = _getServer(environ) @@ -1236,6 +1272,7 @@ def buildinfo(environ, buildID): values['pathinfo'] = pathinfo return _genHTML(environ, 'buildinfo.chtml') + def builds(environ, userID=None, tagID=None, packageID=None, state=None, order='-build_id', start=None, prefix=None, inherited='1', latest='1', type=None): values = _initValues(environ, 'Builds', 'builds') server = _getServer(environ) @@ -1319,6 +1356,7 @@ def builds(environ, userID=None, tagID=None, packageID=None, state=None, order=' return _genHTML(environ, 'builds.chtml') + def users(environ, order='name', start=None, prefix=None): values = _initValues(environ, 'Users', 'users') server = _getServer(environ) @@ -1338,6 +1376,7 @@ def users(environ, order='name', start=None, prefix=None): return _genHTML(environ, 'users.chtml') + def userinfo(environ, userID, packageOrder='package_name', packageStart=None, buildOrder='-completion_time', buildStart=None): values = _initValues(environ, 'User Info', 'users') server = _getServer(environ) @@ -1360,6 +1399,7 @@ def userinfo(environ, userID, packageOrder='package_name', packageStart=None, bu return _genHTML(environ, 'userinfo.chtml') + def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-id', buildrootStart=None): values = _initValues(environ, 'RPM Info', 'builds') server = _getServer(environ) @@ -1413,6 +1453,7 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i return _genHTML(environ, 'rpminfo.chtml') + def archiveinfo(environ, archiveID, fileOrder='name', fileStart=None, buildrootOrder='-id', buildrootStart=None): values = _initValues(environ, 'Archive Info', 'builds') server = _getServer(environ) @@ -1451,6 +1492,7 @@ def archiveinfo(environ, archiveID, fileOrder='name', fileStart=None, buildrootO return _genHTML(environ, 'archiveinfo.chtml') + def fileinfo(environ, filename, rpmID=None, archiveID=None): values = _initValues(environ, 'File Info', 'builds') server = _getServer(environ) @@ -1485,6 +1527,7 @@ def fileinfo(environ, filename, rpmID=None, archiveID=None): return _genHTML(environ, 'fileinfo.chtml') + def cancelbuild(environ, buildID): server = _getServer(environ) _assertLogin(environ) @@ -1500,6 +1543,7 @@ def cancelbuild(environ, buildID): _redirect(environ, 'buildinfo?buildID=%i' % build['id']) + def hosts(environ, state='enabled', start=None, order='name'): values = _initValues(environ, 'Hosts', 'hosts') server = _getServer(environ) @@ -1530,6 +1574,7 @@ def hosts(environ, state='enabled', start=None, order='name'): return _genHTML(environ, 'hosts.chtml') + def hostinfo(environ, hostID=None, userID=None): values = _initValues(environ, 'Host Info', 'hosts') server = _getServer(environ) @@ -1570,6 +1615,7 @@ def hostinfo(environ, hostID=None, userID=None): return _genHTML(environ, 'hostinfo.chtml') + def hostedit(environ, hostID): server = _getServer(environ) _assertLogin(environ) @@ -1619,6 +1665,7 @@ def hostedit(environ, hostID): return _genHTML(environ, 'hostedit.chtml') + def disablehost(environ, hostID): server = _getServer(environ) _assertLogin(environ) @@ -1629,6 +1676,7 @@ def disablehost(environ, hostID): _redirect(environ, 'hostinfo?hostID=%i' % host['id']) + def enablehost(environ, hostID): server = _getServer(environ) _assertLogin(environ) @@ -1639,6 +1687,7 @@ def enablehost(environ, hostID): _redirect(environ, 'hostinfo?hostID=%i' % host['id']) + def channelinfo(environ, channelID): values = _initValues(environ, 'Channel Info', 'hosts') server = _getServer(environ) @@ -1665,6 +1714,7 @@ def channelinfo(environ, channelID): return _genHTML(environ, 'channelinfo.chtml') + def buildrootinfo(environ, buildrootID, builtStart=None, builtOrder=None, componentStart=None, componentOrder=None): values = _initValues(environ, 'Buildroot Info', 'hosts') server = _getServer(environ) @@ -1688,6 +1738,7 @@ def buildrootinfo(environ, buildrootID, builtStart=None, builtOrder=None, compon return _genHTML(environ, template) + def rpmlist(environ, type, buildrootID=None, imageID=None, start=None, order='nvr'): """ rpmlist requires a buildrootID OR an imageID to be passed in. From one @@ -1739,6 +1790,7 @@ def rpmlist(environ, type, buildrootID=None, imageID=None, start=None, order='nv return _genHTML(environ, 'rpmlist.chtml') + def archivelist(environ, type, buildrootID=None, imageID=None, start=None, order='filename'): values = _initValues(environ, 'Archive List', 'hosts') server = _getServer(environ) @@ -1777,6 +1829,7 @@ def archivelist(environ, type, buildrootID=None, imageID=None, start=None, order return _genHTML(environ, 'archivelist.chtml') + def buildtargets(environ, start=None, order='name'): values = _initValues(environ, 'Build Targets', 'buildtargets') server = _getServer(environ) @@ -1792,6 +1845,7 @@ def buildtargets(environ, start=None, order='name'): return _genHTML(environ, 'buildtargets.chtml') + def buildtargetinfo(environ, targetID=None, name=None): values = _initValues(environ, 'Build Target Info', 'buildtargets') server = _getServer(environ) @@ -1821,6 +1875,7 @@ def buildtargetinfo(environ, targetID=None, name=None): return _genHTML(environ, 'buildtargetinfo.chtml') + def buildtargetedit(environ, targetID): server = _getServer(environ) _assertLogin(environ) @@ -1860,6 +1915,7 @@ def buildtargetedit(environ, targetID): return _genHTML(environ, 'buildtargetedit.chtml') + def buildtargetcreate(environ): server = _getServer(environ) _assertLogin(environ) @@ -1894,6 +1950,7 @@ def buildtargetcreate(environ): return _genHTML(environ, 'buildtargetedit.chtml') + def buildtargetdelete(environ, targetID): server = _getServer(environ) _assertLogin(environ) @@ -1908,11 +1965,13 @@ def buildtargetdelete(environ, targetID): _redirect(environ, 'buildtargets') + def reports(environ): _getServer(environ) _initValues(environ, 'Reports', 'reports') return _genHTML(environ, 'reports.chtml') + def buildsbyuser(environ, start=None, order='-builds'): values = _initValues(environ, 'Builds by User', 'reports') server = _getServer(environ) @@ -1940,6 +1999,7 @@ def buildsbyuser(environ, start=None, order='-builds'): return _genHTML(environ, 'buildsbyuser.chtml') + def rpmsbyhost(environ, start=None, order=None, hostArch=None, rpmArch=None): values = _initValues(environ, 'RPMs by Host', 'reports') server = _getServer(environ) @@ -1981,6 +2041,7 @@ def rpmsbyhost(environ, start=None, order=None, hostArch=None, rpmArch=None): return _genHTML(environ, 'rpmsbyhost.chtml') + def packagesbyuser(environ, start=None, order=None): values = _initValues(environ, 'Packages by User', 'reports') server = _getServer(environ) @@ -2010,6 +2071,7 @@ def packagesbyuser(environ, start=None, order=None): return _genHTML(environ, 'packagesbyuser.chtml') + def tasksbyhost(environ, start=None, order='-tasks', hostArch=None): values = _initValues(environ, 'Tasks by Host', 'reports') server = _getServer(environ) @@ -2046,6 +2108,7 @@ def tasksbyhost(environ, start=None, order='-tasks', hostArch=None): return _genHTML(environ, 'tasksbyhost.chtml') + def tasksbyuser(environ, start=None, order='-tasks'): values = _initValues(environ, 'Tasks by User', 'reports') server = _getServer(environ) @@ -2074,6 +2137,7 @@ def tasksbyuser(environ, start=None, order='-tasks'): return _genHTML(environ, 'tasksbyuser.chtml') + def buildsbystatus(environ, days='7'): values = _initValues(environ, 'Builds by Status', 'reports') server = _getServer(environ) @@ -2109,6 +2173,7 @@ def buildsbystatus(environ, days='7'): return _genHTML(environ, 'buildsbystatus.chtml') + def buildsbytarget(environ, days='7', start=None, order='-builds'): values = _initValues(environ, 'Builds by Target', 'reports') server = _getServer(environ) @@ -2148,12 +2213,14 @@ def buildsbytarget(environ, days='7', start=None, order='-builds'): return _genHTML(environ, 'buildsbytarget.chtml') + def _filter_hosts_by_arch(hosts, arch): if arch == '__all__': return hosts else: return [h for h in hosts if arch in h['arches'].split()] + def clusterhealth(environ, arch='__all__'): values = _initValues(environ, 'Cluster health', 'reports') server = _getServer(environ) @@ -2204,6 +2271,7 @@ def clusterhealth(environ, arch='__all__'): values['channels'] = sorted(channels, key=lambda x: x['name']) return _genHTML(environ, 'clusterhealth.chtml') + def recentbuilds(environ, user=None, tag=None, package=None): values = _initValues(environ, 'Recent Build RSS') server = _getServer(environ) @@ -2273,6 +2341,7 @@ def recentbuilds(environ, user=None, tag=None, package=None): environ['koji.headers'].append(['Content-Type', 'text/xml']) return _genHTML(environ, 'recentbuilds.chtml') + _infoURLs = {'package': 'packageinfo?packageID=%(id)i', 'build': 'buildinfo?buildID=%(id)i', 'tag': 'taginfo?tagID=%(id)i', @@ -2299,6 +2368,7 @@ _DEFAULT_SEARCH_ORDER = { # any type not listed will default to 'name' } + def search(environ, start=None, order=None): values = _initValues(environ, 'Search', 'search') server = _getServer(environ) diff --git a/www/kojiweb/wsgi_publisher.py b/www/kojiweb/wsgi_publisher.py index 50e34503..ac5fe944 100644 --- a/www/kojiweb/wsgi_publisher.py +++ b/www/kojiweb/wsgi_publisher.py @@ -248,7 +248,6 @@ class Dispatcher(object): # TODO (warning in header or something?) return func, data - def _setup(self, environ): global kojiweb_handlers global kojiweb diff --git a/www/lib/kojiweb/util.py b/www/lib/kojiweb/util.py index 3233c930..5fa74b12 100644 --- a/www/lib/kojiweb/util.py +++ b/www/lib/kojiweb/util.py @@ -41,6 +41,7 @@ import koji class NoSuchException(Exception): pass + try: # pyOpenSSL might not be around from OpenSSL.SSL import Error as SSL_Error @@ -51,6 +52,7 @@ except: themeInfo = {} themeCache = {} + def _initValues(environ, title='Build System Info', pageID='summary'): global themeInfo global themeCache @@ -69,6 +71,7 @@ def _initValues(environ, title='Build System Info', pageID='summary'): return values + def themePath(path, local=False): global themeInfo global themeCache @@ -95,6 +98,7 @@ def themePath(path, local=False): themeCache[path, local] = ret return ret + class DecodeUTF8(Cheetah.Filters.Filter): def filter(self, *args, **kw): """Convert all strs to unicode objects""" @@ -106,6 +110,8 @@ class DecodeUTF8(Cheetah.Filters.Filter): return result # Escape ampersands so the output can be valid XHTML + + class XHTMLFilter(DecodeUTF8): def filter(self, *args, **kw): result = super(XHTMLFilter, self).filter(*args, **kw) @@ -116,8 +122,10 @@ class XHTMLFilter(DecodeUTF8): result = result.replace('&gt;', '>') return result + TEMPLATES = {} + def _genHTML(environ, fileName): reqdir = os.path.dirname(environ['SCRIPT_FILENAME']) if os.getcwd() != reqdir: @@ -154,11 +162,13 @@ def _genHTML(environ, fileName): else: return tmpl_inst.respond() + def _truncTime(): now = datetime.datetime.now() # truncate to the nearest 15 minutes return now.replace(minute=(now.minute // 15 * 15), second=0, microsecond=0) + def _genToken(environ, tstamp=None): if 'koji.currentLogin' in environ and environ['koji.currentLogin']: user = environ['koji.currentLogin'] @@ -171,6 +181,7 @@ def _genToken(environ, tstamp=None): value = value.encode('utf-8') return hashlib.md5(value).hexdigest()[-8:] + def _getValidTokens(environ): tokens = [] now = _truncTime() @@ -181,6 +192,7 @@ def _getValidTokens(environ): tokens.append(token) return tokens + def toggleOrder(template, sortKey, orderVar='order'): """ If orderVar equals 'sortKey', return '-sortKey', else @@ -191,6 +203,7 @@ def toggleOrder(template, sortKey, orderVar='order'): else: return sortKey + def toggleSelected(template, var, option, checked=False): """ If the passed in variable var equals the literal value in option, @@ -206,6 +219,7 @@ def toggleSelected(template, var, option, checked=False): else: return '' + def sortImage(template, sortKey, orderVar='order'): """ Return an html img tag suitable for inclusion in the sortKey of a sortable table, @@ -219,6 +233,7 @@ def sortImage(template, sortKey, orderVar='order'): else: return '' + def passthrough(template, *vars): """ Construct a string suitable for use as URL @@ -239,6 +254,7 @@ def passthrough(template, *vars): else: return '' + def passthrough_except(template, *exclude): """ Construct a string suitable for use as URL @@ -255,6 +271,7 @@ def passthrough_except(template, *exclude): passvars.append(var) return passthrough(template, *passvars) + def sortByKeyFuncNoneGreatest(key): """Return a function to sort a list of maps by the given key. None will sort higher than all other values (instead of lower). @@ -265,6 +282,7 @@ def sortByKeyFuncNoneGreatest(key): return (v is None, v) return internal_key + def paginateList(values, data, start, dataName, prefix=None, order=None, noneGreatest=False, pageSize=50): """ Slice the 'data' list into one page worth. Start at offset @@ -296,6 +314,7 @@ def paginateList(values, data, start, dataName, prefix=None, order=None, noneGre return data + def paginateMethod(server, values, methodName, args=None, kw=None, start=None, dataName=None, prefix=None, order=None, pageSize=50): """Paginate the results of the method with the given name when called with the given args and kws. @@ -324,6 +343,7 @@ def paginateMethod(server, values, methodName, args=None, kw=None, return data + def paginateResults(server, values, methodName, args=None, kw=None, start=None, dataName=None, prefix=None, order=None, pageSize=50): """Paginate the results of the method with the given name when called with the given args and kws. @@ -352,6 +372,7 @@ def paginateResults(server, values, methodName, args=None, kw=None, return data + def _populateValues(values, dataName, prefix, data, totalRows, start, count, pageSize, order): """Populate the values list with the data about the list provided.""" values[dataName] = data @@ -372,21 +393,25 @@ def _populateValues(values, dataName, prefix, data, totalRows, start, count, pag pages = [page for page in range(0, totalPages) if (abs(page - currentPage) < 100 or ((page + 1) % 100 == 0))] values[(prefix and prefix + 'Pages') or 'pages'] = pages + def stateName(stateID): """Convert a numeric build state into a readable name.""" return koji.BUILD_STATES[stateID].lower() + def imageTag(name): """Return an img tag that loads an icon with the given name""" return '%s' \ % (themePath("images/%s.png" % name), name, name) + def stateImage(stateID): """Return an IMG tag that loads an icon appropriate for the given state""" name = stateName(stateID) return imageTag(name) + def brStateName(stateID): """Convert a numeric buildroot state into a readable name.""" if stateID is None: @@ -414,14 +439,17 @@ def repoStateName(stateID): else: return 'unknown' + def taskState(stateID): """Convert a numeric task state into a readable name""" return koji.TASK_STATES[stateID].lower() + formatTime = koji.formatTime formatTimeRSS = koji.formatTimeLong formatTimeLong = koji.formatTimeLong + def formatTimestampDifference(start_ts, end_ts): diff = end_ts - start_ts seconds = diff % 60 @@ -431,6 +459,7 @@ def formatTimestampDifference(start_ts, end_ts): hours = diff return "%d:%02d:%02d" % (hours, minutes, seconds) + def formatDep(name, version, flags): """Format dependency information into a human-readable format. Copied from @@ -451,6 +480,7 @@ def formatDep(name, version, flags): s = "%s %s" % (s, version) return s + def formatMode(mode): """Format a numeric mode into a ls-like string describing the access mode.""" if stat.S_ISREG(mode): @@ -485,9 +515,11 @@ def formatMode(mode): return result + def formatThousands(value): return '{:,}'.format(value) + def rowToggle(template): """If the value of template._rowNum is even, return 'row-even'; if it is odd, return 'row-odd'. Increment the value before checking it. @@ -529,6 +561,7 @@ _fileFlags = {1: 'configuration', 1024: 'unpatched', 2048: 'public key'} + def formatFileFlags(flags): """Format rpm fileflags for display. Returns a list of human-readable strings specifying the @@ -539,6 +572,7 @@ def formatFileFlags(flags): results.append(desc) return results + def escapeHTML(value): """Replace special characters to the text can be displayed in an HTML page correctly. @@ -554,6 +588,7 @@ def escapeHTML(value): replace('<', '<').\ replace('>', '>') + def authToken(template, first=False, form=False): """Return the current authToken if it exists. If form is True, return it enclosed in a hidden input field. @@ -571,6 +606,7 @@ def authToken(template, first=False, form=False): else: return '' + def explainError(error): """Explain an exception in user-consumable terms @@ -643,6 +679,7 @@ class TaskResultFragment(object): - composer - empty_str_placeholder """ + def __init__(self, text='', size=None, need_escape=None, begin_tag='', end_tag='', composer=None, empty_str_placeholder=None): self.text = text @@ -688,6 +725,7 @@ class TaskResultLine(object): - end_tag - composer """ + def __init__(self, fragments=None, need_escape=None, begin_tag='', end_tag='
', composer=None): if fragments is None: @@ -755,6 +793,7 @@ def _parse_value(key, value, sep=', '): return TaskResultFragment(text=_str, need_escape=need_escape, begin_tag=begin_tag, end_tag=end_tag) + def task_result_to_html(result=None, exc_class=None, max_abbr_lines=None, max_abbr_len=None, abbr_postscript=None):