support for complete history query (not just tag operations)

This commit is contained in:
Mike McLean 2010-08-06 16:12:55 -04:00
parent c43bfe03d0
commit 1893c7cd4a
2 changed files with 533 additions and 0 deletions

287
cli/koji
View file

@ -3364,6 +3364,293 @@ def anon_handle_list_tag_history(options, session, args):
print "%r" % x
print _histline(event_id, x)
def _print_histline(entry, **kwargs):
options = kwargs['options']
event_id, table, create, x = entry
who = None
edit = x.get('.related')
if edit:
del x['.related']
bad_edit = None
if len(edit) != 1:
bad_edit = "%i elements" % len(edit)+1
other = edit[0]
#check edit for sanity
if create or not other[2]:
bad_edit = "out of order"
if event_id != other[0]:
bad_edit = "non-matching"
if bad_edit:
print "Warning: unusual edit at event %i in table %s (%s)" % (event_id, table, bad_edit)
#we'll simply treat them as separate events
pprint.pprint(entry)
pprint.pprint(edit)
_print_histline(entry, **kwargs)
for data in edit:
_print_histline(entry, **kwargs)
return
if create:
ts = x['create_ts']
if x.has_key('creator_name'):
who = "by %(creator_name)s"
else:
ts = x['revoke_ts']
if x.has_key('revoker_name'):
who = "by %(revoker_name)s"
if table == 'tag_listing':
if edit:
fmt = "%(name)s-%(version)s-%(release)s re-tagged into %(tag.name)s"
elif create:
fmt = "%(name)s-%(version)s-%(release)s tagged into %(tag.name)s"
else:
fmt = "%(name)s-%(version)s-%(release)s untagged from %(tag.name)s"
elif table == 'user_perms':
if edit:
fmt = "permission %(permission.name)s re-granted to %(user.name)s"
elif create:
fmt = "permission %(permission.name)s granted to %(user.name)s"
else:
fmt = "permission %(permission.name)s revoked for %(user.name)s"
elif table == 'user_groups':
if edit:
fmt = "user %(user.name)s re-added to group %(group.name)s"
elif create:
fmt = "user %(user.name)s added to group %(group.name)s"
else:
fmt = "user %(user.name)s removed from group %(group.name)s"
elif table == 'tag_packages':
if edit:
fmt = "package list entry for %(package.name)s in %(tag.name)s updated"
elif create:
fmt = "package list entry created: %(package.name)s in %(tag.name)s"
else:
fmt = "package list entry revoked: %(package.name)s in %(tag.name)s"
elif table == 'tag_inheritance':
if edit:
fmt = "inheritance line %(tag.name)s->%(parent.name)s updated"
elif create:
fmt = "inheritance line %(tag.name)s->%(parent.name)s added"
else:
fmt = "inheritance line %(tag.name)s->%(parent.name)s removed"
elif table == 'tag_config':
if edit:
fmt = "tag configuration for %(tag.name)s altered"
elif create:
fmt = "new tag: %(tag.name)s"
else:
fmt = "tag deleted: %(tag.name)s"
elif table == 'build_target_config':
if edit:
fmt = "build target configuration for %(build_target.name)s updated"
elif create:
fmt = "new build target: %(build_target.name)s"
else:
fmt = "build target deleted: %(build_target.name)s"
elif table == 'external_repo_config':
if edit:
fmt = "external repo configuration for %(external_repo.name)s altered"
elif create:
fmt = "new external repo: %(external_repo.name)s"
else:
fmt = "external repo deleted: %(external_repo.name)s"
elif table == 'tag_external_repos':
if edit:
fmt = "external repo entry for %(external_repo.name)s in tag %(tag.name)s updated"
elif create:
fmt = "external repo entry for %(external_repo.name)s added to tag %(tag.name)s"
else:
fmt = "external repo entry for %(external_repo.name)s removed from tag %(tag.name)s"
elif table == 'group_config':
if edit:
fmt = "group %(group.name)s configuration for tag %(tag.name)s updated"
elif create:
fmt = "group %(group.name)s added to tag %(tag.name)s"
else:
fmt = "group %(group.name)s removed from tag %(tag.name)s"
elif table == 'group_req_listing':
if edit:
fmt = "group dependency %(group.name)s->%(req.name)s updated in tag %(tag.name)s"
elif create:
fmt = "group dependency %(group.name)s->%(req.name)s added in tag %(tag.name)s"
else:
fmt = "group dependency %(group.name)s->%(req.name)s dropped from tag %(tag.name)s"
elif table == 'group_package_listing':
if edit:
fmt = "package entry %(package)s in group %(group.name)s, tag %(tag.name)s updated"
elif create:
fmt = "package %(package)s added to group %(group.name)s in tag %(tag.name)s"
else:
fmt = "package %(package)s removed from group %(group.name)s in tag %(tag.name)s"
else:
if edit:
fmt = "%s entry updated" % table
elif create:
fmt = "%s entry created" % table
else:
fmt = "%s entry revoked" % table
time_str = time.asctime(time.localtime(ts))
parts = [time_str, fmt % x]
if options.events or options.verbose:
parts.insert(1, "(eid %i)" % event_id)
if who:
parts.append(who % x)
if create and x['active']:
parts.append("[still active]")
print ' '.join(parts)
hidden_fields = ['active', 'create_event', 'revoke_event', 'creator_id', 'revoker_id',
'creator_name', 'revoker_name', 'create_ts', 'revoke_ts']
def get_nkey(key):
if key == 'perm_id':
return 'permission.name'
elif key.endswith('_id'):
return '%s.name' % key[:-3]
else:
return '%s.name' % key
if edit:
keys = x.keys()
keys.sort()
y = other[-1]
for key in keys:
if key in hidden_fields:
continue
if x[key] == y[key]:
continue
if key[0] == '_':
continue
nkey = get_nkey(key)
if nkey in x and nkey in y:
continue
print " %s: %s -> %s" % (key, x[key], y[key])
elif create and options.verbose and table != 'tag_listing':
keys = x.keys()
keys.sort()
# the table keys have already been represented in the base format string
also_hidden = list(_table_keys[table])
also_hidden.extend([get_nkey(k) for k in also_hidden])
for key in keys:
if key in hidden_fields or key in also_hidden:
continue
nkey = get_nkey(key)
if nkey in x:
continue
if key[0] == '_':
continue
if x.get('blocked') and key != 'blocked':
continue
if key.endswith('.name'):
dkey = key[:-5]
else:
dkey = key
print " %s: %s" % (dkey, x[key])
_table_keys = {
'user_perms' : ['user_id', 'perm_id'],
'user_groups' : ['user_id', 'group_id'],
'tag_inheritance' : ['tag_id', 'parent_id'],
'tag_config' : ['tag_id'],
'build_target_config' : ['build_target_id'],
'external_repo_config' : ['external_repo_id'],
'tag_external_repos' : ['tag_id', 'external_repo_id'],
'tag_listing' : ['build_id', 'tag_id'],
'tag_packages' : ['package_id', 'tag_id'],
'group_config' : ['group_id', 'tag_id'],
'group_req_listing' : ['group_id', 'tag_id', 'req_id'],
'group_package_listing' : ['group_id', 'tag_id', 'package'],
}
def anon_handle_list_history(options, session, args):
"Display historical data"
usage = _("usage: %prog list-tag-history [options]")
usage += _("\n(Specify the --help global option for a list of other help options)")
parser = OptionParser(usage=usage)
parser.add_option("--debug", action="store_true")
parser.add_option("--build", help=_("Only show data for a specific build"))
parser.add_option("--package", help=_("Only show data for a specific package"))
parser.add_option("--tag", help=_("Only show data for a specific tag"))
parser.add_option("--editor", "--by", metavar="USER", help=_("Only show entries modified by user"))
parser.add_option("--user", help=_("Only show entries affecting a user"))
parser.add_option("--permission", help=_("Only show entries relating to a given permission"))
parser.add_option("--external-repo", "--erepo", help=_("Only show entries relating to a given external repo"))
parser.add_option("--build-target", "--target", help=_("Only show entries relating to a given build target"))
parser.add_option("--group", help=_("Only show entries relating to a given group"))
parser.add_option("--before", metavar="TIMESTAMP", help=_("Only show entries before timestamp"))
parser.add_option("--after", metavar="TIMESTAMP", help=_("Only show entries after timestamp"))
parser.add_option("--before-event", metavar="EVENT_ID", type='int', help=_("Only show entries before event"))
parser.add_option("--after-event", metavar="EVENT_ID", type='int', help=_("Only show entries after event"))
parser.add_option("--context", action="store_true", help=_("Show related entries"))
parser.add_option("-s", "--show", action="append", help=_("Show data from selected tables"))
parser.add_option("-v", "--verbose", action="store_true", help=_("Show more detail"))
parser.add_option("-e", "--events", action="store_true", help=_("Show event ids"))
parser.add_option("--all", action="store_true", help=_("Allows listing the entire global history"))
(options, args) = parser.parse_args(args)
if len(args) != 0:
parser.error(_("This command takes no arguments"))
assert False
kwargs = {}
limited = False
for opt in 'package', 'tag', 'build', 'editor', 'user', 'permission', 'external_repo', \
'build_target', 'group', 'before', 'after':
val = getattr(options, opt)
if val:
kwargs[opt] = val
limited = True
if options.before_event:
kwargs['beforeEvent'] = options.before_event
if options.after_event:
kwargs['afterEvent'] = options.after_event
tables = None
if options.show:
tables = []
for arg in options.show:
tables.extend(arg.split(','))
if not limited and not options.all:
parser.error(_("Please specify an option to limit the query"))
activate_session(session)
histdata = session.queryHistory(tables=tables, **kwargs)
timeline = []
def distinguish_match(x, name):
"""determine if create or revoke event matched"""
if options.context:
return True
name = "_" + name
ret = True
for key in x:
if key.startswith(name):
ret = ret and x[key]
return ret
for table in histdata:
hist = histdata[table]
for x in hist:
if x['revoke_event'] is not None:
if distinguish_match(x, 'revoked'):
timeline.append((x['revoke_event'], table, 0, x.copy()))
#pprint.pprint(timeline[-1])
if distinguish_match(x, 'created'):
timeline.append((x['create_event'], table, 1, x))
timeline.sort()
#group edits together
new_timeline = []
last_event = None
edit_index = {}
for entry in timeline:
event_id, table, create, x = entry
if event_id != last_event:
edit_index = {}
last_event = event_id
key = tuple([x[k] for k in _table_keys[table]])
prev = edit_index.get((table, event_id), {}).get(key)
if prev:
prev[-1].setdefault('.related', []).append(entry)
else:
edit_index.setdefault((table, event_id), {})[key] = entry
new_timeline.append(entry)
for entry in new_timeline:
if options.debug:
print "%r" % list(entry)
_print_histline(entry, options=options)
def _parseTaskParams(session, method, task_id):
"""Parse the return of getTaskRequest()"""
params = session.getTaskRequest(task_id)

View file

@ -4571,6 +4571,251 @@ def write_signed_rpm(an_rpm, sigkey, force=False):
koji.splice_rpm_sighdr(sighdr, rpm_path, signedpath)
def query_history(tables=None, **kwargs):
"""Returns history data from various tables that support it
tables: list of versioned tables to search, no value implies all tables
valid entries: user_perms, user_groups, tag_inheritance, tag_config,
build_target_config, external_repo_config, tag_external_repos,
tag_listing, tag_packages, group_config, group_req_listing,
group_package_listing
- Time options -
times are specified as an integer event or a string timestamp
time options are valid for all record types
before: either created or revoked before timestamp
after: either created or revoked after timestamp
beforeEvent: either created or revoked before event id
afterEvent: either created or revoked after event id
- other versioning options-
active: select by active status
editor: record created or revoked by user
- table-specific search options -
use of these options will implicitly limit the search to applicable tables
package: only for given package
build: only for given build
tag: only for given tag
user: only affecting a given user
permission: only relating to a given permission
external_repo: only relateing to an external repo
build_target: only relating to a build target
group: only relating to a (comps) group)
"""
common_fields = {
#fields:aliases common to all versioned tables
'active' : 'active',
'create_event' : 'create_event',
'revoke_event' : 'revoke_event',
'creator_id' : 'creator_id',
'revoker_id' : 'revoker_id',
}
common_joins = [
"events AS ev1 ON ev1.id = create_event",
"LEFT OUTER JOIN events AS ev2 ON ev2.id = revoke_event",
"users AS creator ON creator.id = creator_id",
"LEFT OUTER JOIN users AS revoker ON revoker.id = revoker_id",
]
common_joined_fields = {
'creator.name' : 'creator_name',
'revoker.name' : 'revoker_name',
'EXTRACT(EPOCH FROM ev1.time) AS create_ts' : 'create_ts',
'EXTRACT(EPOCH FROM ev2.time) AS revoke_ts' : 'revoke_ts',
}
table_fields = {
'user_perms' : ['user_id', 'perm_id'],
'user_groups' : ['user_id', 'group_id'],
'tag_inheritance' : ['tag_id', 'parent_id', 'priority', 'maxdepth', 'intransitive', 'noconfig', 'pkg_filter'],
'tag_config' : ['tag_id', 'arches', 'perm_id', 'locked', 'maven_support', 'maven_include_all'],
'build_target_config' : ['build_target_id', 'build_tag', 'dest_tag'],
'external_repo_config' : ['external_repo_id', 'url'],
'tag_external_repos' : ['tag_id', 'external_repo_id', 'priority'],
'tag_listing' : ['build_id', 'tag_id'],
'tag_packages' : ['package_id', 'tag_id', 'owner', 'blocked', 'extra_arches'],
'group_config' : ['group_id', 'tag_id', 'blocked', 'exported', 'display_name', 'is_default', 'uservisible',
'description', 'langonly', 'biarchonly'],
'group_req_listing' : ['group_id', 'tag_id', 'req_id', 'blocked', 'type', 'is_metapkg'],
'group_package_listing' : ['group_id', 'tag_id', 'package', 'blocked', 'type', 'basearchonly', 'requires'],
}
name_joins = {
#joins triggered by table fields for name lookup
#field : [table, join-alias, alias]
'user_id' : ['users', 'users', 'user'],
'perm_id' : ['permissions', 'permission'],
#group_id is overloaded (special case below)
'tag_id' : ['tag'],
'parent_id' : ['tag', 'parent'],
'build_target_id' : ['build_target'],
'build_tag' : ['tag', 'build_tag'],
'dest_tag' : ['tag', 'dest_tag'],
'external_repo_id' : ['external_repo'],
# build_id is special cased
'package_id' : ['package'],
'owner' : ['users', 'owner'],
'req_id' : ['groups', 'req'],
}
if tables is None:
tables = table_fields.keys()
tables.sort()
else:
for table in tables:
if table not in table_fields:
raise koji.GenericError, "Unknown history table: %s" % table
ret = {}
for table in tables:
fields = {}
for field in common_fields:
fullname = "%s.%s" % (table, field)
fields[fullname] = common_fields[field]
joins = list(common_joins)
fields.update(common_joined_fields)
joined = {}
for field in table_fields[table]:
fullname = "%s.%s" % (table,field)
fields[fullname] = field
name_join = name_joins.get(field)
if name_join:
tbl = join_as = name_join[0]
if len(name_join) > 1:
join_as = name_join[1]
joined[tbl] = join_as
fullname = "%s.name" % join_as
if len(name_join) > 2:
#apply alias
fields[fullname] = "%s.name" % name_join[2]
else:
fields[fullname] = fullname
if join_as == tbl:
joins.append('LEFT OUTER JOIN %s ON %s = %s.id' % (tbl, field, tbl))
else:
joins.append('LEFT OUTER JOIN %s AS %s ON %s = %s.id' % (tbl, join_as, field, join_as))
elif field == 'build_id':
#special case
fields.update({
'package.name' : 'name', #XXX?
'build.version' : 'version',
'build.release' : 'release',
'build.epoch' : 'epoch',
'build.state' : 'build.state',
})
joins.extend([
'build ON build_id = build.id',
'package ON build.pkg_id = package.id',
])
joined['build'] = 'build'
joined['package'] = 'package'
elif field == 'group_id':
if table.startswith('group_'):
fields['groups.name'] = 'group.name'
joins.append('groups ON group_id = groups.id')
joined['groups'] = 'groups'
elif table == 'user_groups':
fields['usergroup.name'] = 'group.name'
joins.append('users AS usergroup ON group_id = usergroup.id')
joined['users'] = 'usergroup'
clauses = []
skip = False
data = {}
for arg in kwargs:
value = kwargs[arg]
if arg == 'tag':
if 'tag' not in joined:
skip = True
break
data['tag_id'] = get_tag_id(value, strict=True)
if table == 'tag_inheritance':
#special cased because there are two tag columns
clauses.append("tag_id = %(tag_id)i OR parent_id = %(tag_id)i")
else:
clauses.append("%s.id = %%(tag_id)i" % joined['tag'])
elif arg == 'build':
if 'build' not in joined:
skip = True
break
data['build_id'] = get_build(value, strict=True)['id']
clauses.append("build.id = %(build_id)i")
elif arg == 'package':
if 'package' not in joined:
skip = True
break
data['pkg_id'] = get_package_id(value, strict=True)
clauses.append("package.id = %(pkg_id)i")
elif arg == 'user':
if 'users' not in joined:
skip = True
break
data['affected_user_id'] = get_user(value, strict=True)['id']
clauses.append("%s.id = %%(affected_user_id)i" % joined['users'])
elif arg == 'permission':
if 'permissions' not in joined:
skip = True
break
data['perm_id'] = get_perm_id(value, strict=True)
clauses.append("%s.id = %%(perm_id)i" % joined['permissions'])
elif arg == 'external_repo':
if 'external_repo' not in joined:
skip = True
break
data['external_repo_id'] = get_external_repo_id(value, strict=True)
clauses.append("%s.id = %%(external_repo_id)i" % joined['external_repo'])
elif arg == 'build_target':
if 'build_target' not in joined:
skip = True
break
data['build_target_id'] = get_build_target_id(value, strict=True)
clauses.append("%s.id = %%(build_target_id)i" % joined['build_target'])
elif arg == 'group':
if 'groups' not in joined:
skip = True
break
data['group_id'] = get_group_id(value, strict=True)
clauses.append("%s.id = %%(group_id)i" % joined['groups'])
elif arg == 'active':
if value:
clauses.append('active = TRUE')
elif value is not None:
clauses.append('active = FALSE')
elif arg == 'editor':
data['editor'] = get_user(value, strict=True)['id']
clauses.append('creator.id = %(editor)i OR revoker.id = %(editor)i')
fields['creator.id = %(editor)i'] = '_created_by'
fields['revoker.id = %(editor)i'] = '_revoked_by'
elif arg == 'after':
if not isinstance(value, basestring):
value = datetime.datetime.fromtimestamp(value).isoformat(' ')
data['after'] = value
clauses.append('ev1.time > %(after)s OR ev2.time > %(after)s')
fields['ev1.time > %(after)s'] = '_created_after'
fields['ev2.time > %(after)s'] = '_revoked_after'
#clauses.append('EXTRACT(EPOCH FROM ev1.time) > %(after)s OR EXTRACT(EPOCH FROM ev2.time) > %(after)s')
elif arg == 'afterEvent':
data['afterEvent'] = value
clauses.append('create_event > %(afterEvent)i OR revoke_event > %(afterEvent)i')
fields['create_event > %(afterEvent)i'] = '_created_after_event'
fields['revoke_event > %(afterEvent)i'] = '_revoked_after_event'
elif arg == 'before':
if not isinstance(value, basestring):
value = datetime.datetime.fromtimestamp(value).isoformat(' ')
data['before'] = value
clauses.append('ev1.time < %(before)s OR ev2.time < %(before)s')
#clauses.append('EXTRACT(EPOCH FROM ev1.time) < %(before)s OR EXTRACT(EPOCH FROM ev2.time) < %(before)s')
fields['ev1.time < %(before)s'] = '_created_before'
fields['ev2.time < %(before)s'] = '_revoked_before'
elif arg == 'beforeEvent':
data['beforeEvent'] = value
clauses.append('create_event < %(beforeEvent)i OR revoke_event < %(beforeEvent)i')
fields['create_event < %(beforeEvent)i'] = '_created_before_event'
fields['revoke_event < %(beforeEvent)i'] = '_revoked_before_event'
if skip:
continue
fields, aliases = zip(*fields.items())
query = QueryProcessor(columns=fields, aliases=aliases, tables=[table],
joins=joins, clauses=clauses, values=data)
ret[table] = query.execute()
return ret
def tag_history(build=None, tag=None, package=None, queryOpts=None):
"""Returns historical tag data
@ -6423,6 +6668,7 @@ class RootExports(object):
untaggedBuilds = staticmethod(untagged_builds)
tagHistory = staticmethod(tag_history)
queryHistory = staticmethod(query_history)
buildMap = staticmethod(build_map)
deleteBuild = staticmethod(delete_build)