particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
debian-koji/util/koji-gc
Pavol Babincak 9e89ebb9a4 Ran reindent.py from cpython tools on all python scripts 
Citing from reindent docs:

    Change Python (.py) files to use 4-space indents and no hard tab
    characters. Also trim excess spaces and tabs from ends of lines, and
    remove empty lines at the end of files.  Also ensure the last line
    ends with a newline.

Citing from PEP 8:

    Use 4 spaces per indentation level.

    Python 2 code indented with a mixture of tabs and spaces should be
    converted to using spaces exclusively.

    Don't write string literals that rely on significant trailing
    whitespace. Such trailing whitespace is visually indistinguishable
    and some editors (or more recently, reindent.py) will trim them.

Also PyLint recommends not to have trailing whitespace on any line.
2015-05-13 16:39:53 -04:00

959 lines
35 KiB
Python
Executable file
Raw Blame History

#!/usr/bin/python
# koji-gc: a garbage collection tool for Koji
# Copyright (c) 2007-2014 Red Hat, Inc.
#
# Authors:
# Mike McLean <mikem@redhat.com>
try:
import krbV
except ImportError:
pass
import koji
from koji.util import LazyDict, LazyValue
import koji.policy
import ConfigParser
from email.MIMEText import MIMEText
import fnmatch
import optparse
import os
import pprint
import smtplib
import socket # for socket.error
import sys
import time
import xmlrpclib # for ProtocolError and Fault
OptionParser = optparse.OptionParser
if optparse.__version__ == "1.4.1+":
def _op_error(self, msg):
self.print_usage(sys.stderr)
msg = "%s: error: %s\n" % (self._get_prog_name(), msg)
if msg:
sys.stderr.write(msg)
sys.exit(2)
OptionParser.error = _op_error
class MySession(koji.ClientSession):
"""This is a hack to work around server timeouts"""
def _callMethod(self, name, args, kwargs):
retries = 10
i = 0
while True:
i += 1
try:
return super(MySession, self)._callMethod(name, args, kwargs)
except (socket.timeout, socket.error, xmlrpclib.ProtocolError), e:
if i > retries:
raise
else:
print "Socket Error: %s [%i], retrying..." % (e, i)
time.sleep(60)
#an even worse hack
def multiCall(self):
if not self.multicall:
raise GenericError, 'ClientSession.multicall must be set to True before calling multiCall()'
if len(self._calls) == 0:
return []
self.multicall = False
calls = self._calls
self._calls = []
return self._callMethod('multiCall', (calls,), {})
def _(args):
"""Stub function for translation"""
return args
def get_options():
"""process options from command line and config file"""
usage = _("%prog [options]")
parser = OptionParser(usage=usage)
parser.add_option("-c", "--config-file", metavar="FILE",
help=_("use alternate configuration file"))
parser.add_option("--keytab", help=_("specify a Kerberos keytab to use"))
parser.add_option("--principal", help=_("specify a Kerberos principal to use"))
parser.add_option("--krbservice", default="host",
help=_("the service name of the principal being used by the hub"))
parser.add_option("--runas", metavar="USER",
help=_("run as the specified user (requires special privileges)"))
parser.add_option("--user", help=_("specify user"))
parser.add_option("--password", help=_("specify password"))
parser.add_option("--noauth", action="store_true", default=False,
help=_("do not authenticate"))
parser.add_option("--network-hack", action="store_true", default=False,
help=_("enable hackish workaround for broken networks"))
parser.add_option("--cert", default='/etc/koji-gc/client.crt',
help=_("Client SSL certificate file for authentication"))
parser.add_option("--ca", default='/etc/koji-gc/clientca.crt',
help=_("CA cert file that issued the client certificate"))
parser.add_option("--serverca", default='/etc/koji-gc/serverca.crt',
help=_("CA cert file that issued the hub certificate"))
parser.add_option("-n", "--test", action="store_true", default=False,
help=_("test mode"))
parser.add_option("-d", "--debug", action="store_true", default=False,
help=_("show debug output"))
parser.add_option("--debug-xmlrpc", action="store_true", default=False,
help=_("show xmlrpc debug output"))
parser.add_option("--smtp-host", metavar="HOST",
help=_("specify smtp server for notifications"))
parser.add_option("--no-mail", action='store_false', default=True, dest="mail",
help=_("don't send notifications"))
parser.add_option("--send-mail", action='store_true', dest="mail",
help=_("send notifications"))
parser.add_option("--email-domain", default="fedoraproject.org",
help=_("Email domain appended to Koji username for notifications"))
parser.add_option("--from-addr", default="Koji Build System <buildsys@example.com>",
help=_("From address for notifications"))
parser.add_option("--action", help=_("action(s) to take"))
parser.add_option("--delay", metavar="INTERVAL", default = '5 days',
help="time before eligible builds are placed in trashcan")
parser.add_option("--grace-period", default='4 weeks', metavar="INTERVAL",
help="time that builds are held in trashcan")
parser.add_option("--skip-main", action="store_true", default=False,
help=_("don't actually run main"))
parser.add_option("--unprotected-keys", metavar="KEYS",
help=_("allow builds signed with these keys to be deleted"))
parser.add_option("--tag-filter", "--tag", metavar="PATTERN", action="append",
help=_("Process only tags matching PATTERN when pruning"))
parser.add_option("--ignore-tags", metavar="PATTERN", action="append",
help=_("Ignore tags matching PATTERN when pruning"))
parser.add_option("--pkg-filter", "--pkg", "--package",
metavar="PATTERN", action='append',
help=_("Process only packages matching PATTERN"))
parser.add_option("--bypass-locks", metavar="PATTERN", action="append",
help=_("Bypass locks for tags matching PATTERN"))
parser.add_option("--purge", action="store_true", default=False,
help=_("When pruning, attempt to delete the builds that are untagged"))
parser.add_option("--trashcan-tag", default='trashcan', metavar="TAG",
help=_("specify an alternate trashcan tag"))
parser.add_option("--weburl", default="http://localhost/koji", metavar="URL",
help=_("url of koji web server (for use in notifications)"))
parser.add_option("-s", "--server", help=_("url of koji XMLRPC server"))
#parse once to get the config file
(options, args) = parser.parse_args()
defaults = parser.get_default_values()
config = ConfigParser.ConfigParser()
cf = getattr(options, 'config_file', None)
if cf:
if not os.access(cf, os.F_OK):
parser.error(_("No such file: %s") % cf)
assert False
else:
cf = '/etc/koji-gc/koji-gc.conf'
if not os.access(cf, os.F_OK):
cf = None
if not cf:
print "no config file"
config = None
else:
config.read(cf)
#allow config file to update defaults for certain options
cfgmap = [
['keytab', None, 'string'],
['principal', None, 'string'],
['krbservice', None, 'string'],
['runas', None, 'string'],
['user', None, 'string'],
['password', None, 'string'],
['noauth', None, 'boolean'],
['cert', None, 'string'],
['ca', None, 'string'],
['serverca', None, 'string'],
['server', None, 'string'],
['weburl', None, 'string'],
['smtp_host', None, 'string'],
['from_addr', None, 'string'],
['email_domain', None, 'string'],
['mail', None, 'boolean'],
['delay', None, 'string'],
['unprotected_keys', None, 'string'],
['grace_period', None, 'string'],
['trashcan_tag', None, 'string'],
]
for name, alias, type in cfgmap:
if alias is None:
alias = ('main', name)
if config.has_option(*alias):
if options.debug:
print "Using option %s from config file" % (alias,)
if type == 'integer':
setattr(defaults, name, config.getint(*alias))
elif type == 'boolean':
setattr(defaults, name, config.getboolean(*alias))
else:
setattr(defaults, name, config.get(*alias))
#parse again with defaults
(options, args) = parser.parse_args(values=defaults)
options.config = config
#figure out actions
actions = ('prune', 'trash', 'delete', 'salvage')
if options.action:
options.action = options.action.lower().replace(',',' ').split()
for x in options.action:
if x not in actions:
parser.error(_("Invalid action: %s") % x)
else:
options.action = ('delete', 'prune', 'trash')
#split patterns for unprotected keys
if options.unprotected_keys:
options.unprotected_key_patterns = options.unprotected_keys.replace(',',' ').split()
else:
options.unprotected_key_patterns = []
#parse key aliases
options.key_aliases = {}
try:
if config and config.has_option('main', 'key_aliases'):
for line in config.get('main','key_aliases').splitlines():
parts = line.split()
if len(parts) < 2:
continue
options.key_aliases[parts[0].upper()] = parts[1]
except ValueError, e:
print e
parser.error(_("Invalid key alias data in config: %s") % config.get('main','key_aliases'))
#parse time intervals
for key in ('delay', 'grace_period'):
try:
value = getattr(options, key)
value = parse_duration(value)
setattr(options, key, value)
if options.debug:
print "%s: %s seconds" % (key, value)
except ValueError:
parser.error(_("Invalid time interval: %s") % value)
return options, args
def check_tag(name):
"""Check tag name against options and determine if we should process it
The ignore option takes priority here.
Returns True if we should process the tag, False otherwise
"""
if options.ignore_tags:
for pattern in options.ignore_tags:
if fnmatch.fnmatch(name, options.tag_filter):
return False
if options.tag_filter:
for pattern in options.tag_filter:
if fnmatch.fnmatch(name, pattern):
return True
#doesn't match any pattern in filter
return False
else:
#not ignored and no filter specified
return True
def check_package(name):
"""Check package name against options and determine if we should process it
Returns True if we should process the package, False otherwise
"""
if options.pkg_filter:
for pattern in options.pkg_filter:
if fnmatch.fnmatch(name, pattern):
return True
#doesn't match any pattern in filter
return False
else:
#no filter specified
return True
time_units = {
'second' : 1,
'minute' : 60,
'hour' : 3600,
'day' : 86400,
'week' : 604800,
}
time_unit_aliases = [
#[unit, alias, alias, ...]
['week', 'weeks', 'wk', 'wks'],
['hour', 'hours', 'hr', 'hrs'],
['day', 'days'],
['minute', 'minutes', 'min', 'mins'],
['second', 'seconds', 'sec', 'secs', 's'],
]
def parse_duration(str):
"""Parse time duration from string, returns duration in seconds"""
ret = 0
n = None
unit = None
def parse_num(s):
try:
return int(s)
except ValueError:
pass
try:
return float(s)
except ValueError:
pass
return None
for x in str.split():
if n is None:
n = parse_num(x)
if n is not None:
continue
#perhaps the unit is appended w/o a space
for names in time_unit_aliases:
for name in names:
if x.endswith(name):
n = parse_num(x[:-len(name)])
if n is None:
continue
unit = names[0]
# combined at end
break
if unit:
break
else:
raise ValueError, "Invalid time interval: %s" % str
if unit is None:
x = x.lower()
for names in time_unit_aliases:
for name in names:
if x == name:
unit = names[0]
break
if unit:
break
else:
raise ValueError, "Invalid time interval: %s" % str
ret += n * time_units[unit]
n = None
unit = None
return ret
def error(msg=None, code=1):
if msg:
sys.stderr.write(msg + "\n")
sys.stderr.flush()
sys.exit(code)
def warn(msg):
sys.stderr.write(msg + "\n")
sys.stderr.flush()
def ensure_connection(session):
try:
ret = session.getAPIVersion()
except xmlrpclib.ProtocolError:
error(_("Error: Unable to connect to server"))
if ret != koji.API_VERSION:
warn(_("WARNING: The server is at API version %d and the client is at %d" % (ret, koji.API_VERSION)))
def has_krb_creds():
if not sys.modules.has_key('krbV'):
return False
try:
ctx = krbV.default_context()
ccache = ctx.default_ccache()
princ = ccache.principal()
return True
except krbV.Krb5Error:
return False
def activate_session(session):
"""Test and login the session is applicable"""
global options
if options.noauth:
#skip authentication
pass
elif os.path.isfile(options.cert):
# authenticate using SSL client cert
session.ssl_login(options.cert, options.ca, options.serverca, proxyuser=options.runas)
elif options.user:
#authenticate using user/password
session.login()
elif has_krb_creds():
try:
if options.keytab and options.principal:
session.krb_login(principal=options.principal, keytab=options.keytab, proxyuser=options.runas)
else:
session.krb_login(proxyuser=options.runas)
except krbV.Krb5Error, e:
error(_("Kerberos authentication failed: %s (%s)") % (e.args[1], e.args[0]))
except socket.error, e:
warn(_("Could not connect to Kerberos authentication service: '%s'") % e.args[1])
if not options.noauth and not session.logged_in:
error(_("Error: unable to log in, no authentication methods available"))
ensure_connection(session)
if options.debug:
print "successfully connected to hub"
def send_warning_notice(owner_name, builds):
if not options.mail:
return
if not builds:
print "Warning: empty build list. No notice sent"
return
head = """\
The following build(s) are unreferenced and have been marked for
deletion. They will be held in the trashcan tag for a grace period.
At the end of that period they will be deleted permanently. This
garbage collection is a normal part of build system operation.
Please see the following url for more information:
http://fedoraproject.org/wiki/Koji/GarbageCollection"""
fmt="""\
Build: %%(name)s-%%(version)s-%%(release)s
%s/buildinfo?buildID=%%(id)i""" % options.weburl
middle = '\n\n'.join([fmt % b for b in builds])
tail = """\
If you would like to protect any of these builds from deletion, please
refer to the document linked above for instructions."""
msg = MIMEText('\n\n'.join([head, middle, tail]))
if len(builds) == 1:
msg['Subject'] = "1 build marked for deletion"
else:
msg['Subject'] = "%i builds marked for deletion" % len(builds)
msg['From'] = options.from_addr
msg['To'] = "%s@%s" % (owner_name, options.email_domain) #XXX!
msg['X-Koji-Builder'] = owner_name
if options.test:
if options.debug:
print str(msg)
else:
print "Would have sent warning notice to %s" % msg['To']
else:
if options.debug:
print "Sending warning notice to %s" % msg['To']
s = smtplib.SMTP(options.smtp_host)
s.connect()
s.sendmail(msg['From'], msg['To'], msg.as_string())
s.close()
def main(args):
activate_session(session)
for x in options.action:
globals()['handle_' + x]()
def handle_trash():
print "Getting untagged builds..."
untagged = session.untaggedBuilds()
print "...got %i builds" % len(untagged)
min_age = options.delay
trashcan_tag = options.trashcan_tag
#Step 1: place unreferenced builds into trashcan
i = 0
N = len(untagged)
to_trash = []
for binfo in untagged:
i += 1
nvr = "%(name)s-%(version)s-%(release)s" % binfo
if not check_package(binfo['name']):
if options.debug:
print "[%i/%i] Skipping package: %s" % (i, N, nvr)
continue
try:
refs = session.buildReferences(binfo['id'], limit=10)
except xmlrpclib.Fault:
print "[%i/%i] Error checking references for %s. Skipping" % (i, N, nvr)
continue
#XXX - this is more data than we need
# also, this call takes waaaay longer than it should
if refs['tags']:
# must have been tagged just now
print "[%i/%i] Build is tagged [?]: %s" % (i, N, nvr)
continue
if refs['rpms']:
if options.debug:
print "[%i/%i] Build has %i rpm references: %s" % (i, N, len(refs['rpms']), nvr)
#pprint.pprint(refs['rpms'])
continue
if refs['archives']:
if options.debug:
print "[%i/%i] Build has %i archive references: %s" % (i, N, len(refs['archives']), nvr)
#pprint.pprint(refs['archives'])
continue
ts = refs['last_used']
if ts:
#work around server bug
if isinstance(ts, list):
ts = ts[0]
#XXX - should really check time server side
if options.debug:
print "[%i/%i] Build has been used in a buildroot: %s" % (i, N, nvr)
print "Last_used: %r" % ts
age = time.time() - ts
if age < min_age:
continue
#see how long build has been untagged
history = session.tagHistory(build=binfo['id'])
age = None
binfo2 = None
if not history:
#never tagged, we'll have to use the build create time
binfo2 = session.getBuild(binfo['id'])
ts = binfo2.get('creation_ts')
if ts is None:
# older api with no good way to get a proper timestamp for
# a build, so we have the following hack
task_id = binfo2.get('task_id')
if task_id:
tinfo = session.getTaskInfo(task_id)
if tinfo['completion_ts']:
age = time.time() - tinfo['completion_ts']
else:
age = time.time() - ts
else:
history = [(h['revoke_event'],h) for h in history]
last = max(history)[1]
if not last['revoke_event']:
#this might happen if the build was tagged just now
print "[%i/%i] Warning: build not untagged: %s" % (i, N, nvr)
continue
age = time.time() - last['revoke_ts']
if age is not None and age < min_age:
if options.debug:
print "[%i/%i] Build untagged only recently: %s" % (i, N, nvr)
continue
#check build signatures
keys = get_build_sigs(binfo['id'], cache=True)
if keys and options.debug:
print "Build: %s, Keys: %s" % (nvr, keys)
if protected_sig(keys):
print "Skipping build %s. Keys: %s" % (nvr, keys)
continue
#ok, go ahead add it to the list
if binfo2 is None:
binfo2 = session.getBuild(binfo['id'])
binfo2['nvr'] = nvr
print "[%i/%i] Adding build to trash list: %s" % (i, N, nvr)
to_trash.append(binfo2)
#process to_trash
#group by owner so we can reduce the number of notices
by_owner = {}
for binfo in to_trash:
by_owner.setdefault(binfo['owner_name'], []).append(binfo)
owners = by_owner.keys()
owners.sort()
for owner_name in owners:
builds = [(b['nvr'], b) for b in by_owner[owner_name]]
builds.sort()
send_warning_notice(owner_name, [x[1] for x in builds])
for nvr, binfo in builds:
if options.test:
print "Would have moved to trashcan: %s" % nvr
else:
if options.debug:
print "Moving to trashcan: %s" % nvr
#figure out package owner
count = {}
for pkg in session.listPackages(pkgID=binfo['name']):
count.setdefault(pkg['owner_id'], 0)
count[pkg['owner_id']] += 1
if not count:
print "Warning: no owner for %s, using build owner" % nvr
#best we can do currently
owner = binfo['owner_id']
else:
owner = max([(n, k) for k, n in count.iteritems()])[1]
session.packageListAdd(trashcan_tag, binfo['name'], owner)
session.tagBuildBypass(trashcan_tag, binfo['id'], force=True)
def protected_sig(keys):
"""Check list of keys and see if any are protected
returns True if ANY are protected (not on unprotected list)
returns False if ALL are unprotected
"""
for key in keys:
if not key:
continue
if not sigmatch(key, options.unprotected_key_patterns):
#this key is protected
return True
return False
def handle_salvage():
"""Reclaim builds from trashcan
Check builds in trashcan for new tags or references and salvage them
(remove trashcan tag) if appropriate.
The delete action also does this, but this is for when you want to
run this action only."""
return handle_delete(just_salvage=True)
def salvage_build(binfo):
"""Removes trashcan tag from a build and prints a message"""
if options.test:
print "Would have untagged from trashcan: %(nvr)s" % binfo
else:
if options.debug:
print "Untagging from trashcan: %(nvr)s" % binfo
session.untagBuildBypass(options.trashcan_tag, binfo['id'], force=True)
def handle_delete(just_salvage=False):
"""Delete builds that have been in the trashcan for long enough
If just_salvage is True, goes into salvage mode. In salvage mode it only
reclaims referenced builds from the trashcan, it does not perform any
deletes
"""
print "Getting list of builds in trash..."
trashcan_tag = options.trashcan_tag
trash = [(b['nvr'], b) for b in session.listTagged(trashcan_tag)]
trash.sort()
print "...got %i builds" % len(trash)
#XXX - it would be better if there were more appropriate server calls for this
grace_period = options.grace_period
for nvr, binfo in trash:
# see if build has been tagged elsewhere
if not check_package(binfo['name']):
if options.debug:
print "Skipping package: %s" % nvr
continue
tags = [t['name'] for t in session.listTags(build=binfo['id']) if t['name'] != trashcan_tag]
if tags:
print "Build %s tagged elsewhere: %s" % (nvr, tags)
salvage_build(binfo)
continue
#check build signatures
keys = get_build_sigs(binfo['id'], cache=False)
if keys and options.debug:
print "Build: %s, Keys: %s" % (nvr, keys)
if protected_sig(keys):
print "Salvaging signed build %s. Keys: %s" % (nvr, keys)
salvage_build(binfo)
continue
if just_salvage:
# skip the rest when salvaging
continue
# determine how long this build has been in the trashcan
history = session.tagHistory(build=binfo['id'], tag=trashcan_tag)
current = [x for x in history if x['active']]
if not current:
#untagged just now?
print "Warning: history missing for %s" % nvr
pprint.pprint(binfo)
pprint.pprint(history)
continue
assert len(current) == 1 #see db constraint
current = current[0]
age = time.time() - current['create_ts']
if age < grace_period:
if options.debug:
print "Skipping build %s, age=%i" % (nvr, age)
continue
# go ahead and delete
if options.test:
print "Would have deleted build from trashcan: %s" % nvr
else:
print "Deleting build: %s" % nvr
session.untagBuildBypass(trashcan_tag, binfo['id'])
try:
session.deleteBuild(binfo['id'])
except (xmlrpclib.Fault, koji.GenericError), e:
print "Warning: deletion failed: %s" % e
#server issue
pass
#TODO - log details for delete failures
class TagPruneTest(koji.policy.MatchTest):
name = 'tag'
field = 'tagname'
class PackagePruneTest(koji.policy.MatchTest):
name = 'package'
field = 'pkgname'
class VolumePruneTest(koji.policy.MatchTest):
name = 'volume'
field = 'volname'
class SigPruneTest(koji.policy.BaseSimpleTest):
name = 'sig'
def run(self, data):
# true if any of the keys match any of the patterns
patterns = self.str.split()[1:]
for key in data['keys']:
if sigmatch(key, patterns):
return True
return False
def sigmatch(key, patterns):
"""Test whether a key id matches any of the given patterns
Supports key aliases
"""
if not isinstance(patterns, (tuple, list)):
patterns = (patterns,)
for pat in patterns:
if fnmatch.fnmatch(key, pat):
return True
alias = options.key_aliases.get(key.upper())
if alias is not None and fnmatch.fnmatch(alias, pat):
return True
return False
class OrderPruneTest(koji.policy.CompareTest):
name = 'order'
field = 'order'
allow_float = False
class AgePruneTest(koji.policy.BaseSimpleTest):
name = 'age'
cmp_idx = koji.policy.CompareTest.operators
def __init__(self, str):
"""Read the test parameters from string"""
super(AgePruneTest, self).__init__(str)
self.cmp, value = str.split(None, 2)[1:]
self.func = self.cmp_idx.get(self.cmp, None)
if self.func is None:
raise Exception, "Invalid comparison in test: %s" % str
self.span = parse_duration(value)
def run(self, data):
return self.func(time.time() - data['ts'], self.span)
def read_policies(fn=None):
"""Read tag gc policies from file
The expected format as follows
test [params] [&& test [params] ...] :: (keep|untag|skip)
"""
fo = file(fn, 'r')
tests = koji.policy.findSimpleTests(globals())
ret = koji.policy.SimpleRuleSet(fo, tests)
fo.close()
return ret
def scan_policies(str):
"""Read tag gc policies from a string
The expected format as follows
test [params] [&& test [params] ...] :: (keep|untag|skip)
"""
tests = koji.policy.findSimpleTests(globals())
return koji.policy.SimpleRuleSet(str.splitlines(), tests)
build_sig_cache = {}
def get_build_sigs(build, cache=False):
if cache and build in build_sig_cache:
return build_sig_cache[build]
rpms = session.listRPMs(buildID=build)
keys = {}
if not rpms:
# for non-rpm builds we have no easy way of checking signatures
ret = build_sig_cache[build] = []
return ret
else:
#TODO - multicall helps, but it might be good to have a more robust server-side call
session.multicall = True
for rpminfo in rpms:
session.queryRPMSigs(rpm_id=rpminfo['id'])
for rpminfo, [sigs] in zip(rpms, session.multiCall()):
for sig in sigs:
if sig['sigkey']:
keys.setdefault(sig['sigkey'], 1)
ret = build_sig_cache[build] = keys.keys()
return ret
def handle_prune():
"""Untag old builds according to policy
If purge is True, will also attempt to delete the pruned builds afterwards
"""
#read policy
if not options.config or not options.config.has_option('prune', 'policy'):
print "Skipping prune step. No policies available."
return
#policies = read_policies(options.policy_file)
policies = scan_policies(options.config.get('prune', 'policy'))
for action in policies.all_actions():
if action not in ("keep", "untag", "skip"):
raise Exception, "Invalid action: %s" % action
if options.debug:
pprint.pprint(policies.ruleset)
#get tags
tags = session.listTags(queryOpts={'order': 'name'})
untagged = {}
build_ids = {}
for taginfo in tags:
tagname = taginfo['name']
if tagname == options.trashcan_tag:
if options.debug:
print "Skipping trashcan tag: %s" % tagname
continue
if not check_tag(tagname):
#if options.debug:
# print "skipping tag due to filter: %s" % tagname
continue
bypass = False
if taginfo['locked']:
if options.bypass_locks:
for pattern in options.bypass_locks:
if fnmatch.fnmatch(tagname, pattern):
bypass = True
break
if bypass:
print "Bypassing lock on tag: %s" % tagname
else:
if options.debug:
print "skipping locked tag: %s" % tagname
continue
if options.debug:
print "Pruning tag: %s" % tagname
#get builds
history = session.tagHistory(tag=tagname, active=True, queryOpts={'order': '-create_ts'})
if not history:
if options.debug:
print "No history for %s" % tagname
continue
pkghist = {}
for h in history:
if taginfo['maven_include_all'] and h['maven_build_id']:
pkghist.setdefault(h['name'] + '-' + h['version'], []).append(h)
else:
pkghist.setdefault(h['name'], []).append(h)
pkgs = pkghist.keys()
pkgs.sort()
for pkg in pkgs:
if not check_package(pkg):
#if options.debug:
# print "skipping package due to filter: %s" % pkg
continue
if options.debug:
print pkg
hist = pkghist[pkg]
#these are the *active* history entries for tag/pkg
skipped = 0
for order, entry in enumerate(hist):
# get sig data
nvr = "%(name)s-%(version)s-%(release)s" % entry
data = {
'tagname' : tagname,
'pkgname' : pkg,
'order': order - skipped,
'ts' : entry['create_ts'],
'nvr' : nvr,
}
data = LazyDict(data)
data['keys'] = LazyValue(get_build_sigs, (entry['build_id'],), {'cache':True})
data['volname'] = LazyValue(lambda x: session.getBuild(x).get('volume_name'),
(entry['build_id'],), cache=True)
build_ids[nvr] = entry['build_id']
action = policies.apply(data)
if action is None:
if options.debug:
print "No policy for %s (%s)" % (nvr, tagname)
if action == 'skip':
skipped += 1
if options.debug:
print policies.last_rule()
print "%s: %s (%s, %i)" % (action, nvr, tagname, order)
if action == 'untag':
if options.test:
print "Would have untagged %s from %s" % (nvr, tagname)
untagged.setdefault(nvr, {})[tagname] = 1
else:
print "Untagging build %s from %s" % (nvr, tagname)
try:
session.untagBuildBypass(taginfo['id'], entry['build_id'], force=bypass)
untagged.setdefault(nvr, {})[tagname] = 1
except (xmlrpclib.Fault, koji.GenericError), e:
print "Warning: untag operation failed: %s" % e
pass
# if action == 'keep' do nothing
if options.purge and untagged:
print "Attempting to purge %i builds" % len(untagged)
for nvr in untagged:
build_id = build_ids[nvr]
tags = [t['name'] for t in session.listTags(build_id)]
if options.test:
#filted out the tags we would have dropped above
tags = [t for t in tags if t not in untagged[nvr]]
if tags:
#still tagged somewhere
print "Skipping %s, still tagged: %s" % (nvr, tags)
continue
#check cached sigs first to save a little time
if build_id in build_sig_cache:
keys = build_sig_cache[build_id]
if protected_sig(keys):
print "Skipping %s, signatures: %s" % (nvr, keys)
continue
#recheck signatures in case build was signed during run
keys = get_build_sigs(build_id, cache=False)
if protected_sig(keys):
print "Skipping %s, signatures: %s" % (nvr, keys)
continue
if options.test:
print "Would have deleted build: %s" % nvr
else:
print "Deleting untagged build: %s" % nvr
try:
session.deleteBuild(build_id, strict=False)
except (xmlrpclib.Fault, koji.GenericError), e:
print "Warning: deletion failed: %s" % e
#server issue
pass
if __name__ == "__main__":
options, args = get_options()
session_opts = {}
for k in ('user', 'password', 'krbservice', 'email_domain', 'debug_xmlrpc', 'debug'):
session_opts[k] = getattr(options,k)
if options.network_hack:
socket.setdefaulttimeout(180)
session = MySession(options.server, session_opts)
else:
session = koji.ClientSession(options.server, session_opts)
rv = 0
try:
if not options.skip_main:
rv = main(args)
if not rv:
rv = 0
except KeyboardInterrupt:
pass
except SystemExit:
rv = 1
#except:
# if options.debug:
# raise
# else:
# exctype, value = sys.exc_info()[:2]
# rv = 1
# print "%s: %s" % (exctype, value)
try:
session.logout()
except:
pass
if not options.skip_main:
sys.exit(rv)
Reference in a new issue View git blame Copy permalink