diff --git a/util/koji-gc b/util/koji-gc index 9ff8a5c4..7fea4c9e 100755 --- a/util/koji-gc +++ b/util/koji-gc @@ -483,7 +483,7 @@ def handle_trash(): N = len(untagged) to_trash = [] - print("1st pass: blacklist") + print("1st pass: package filter") continuing = [] for binfo in untagged: i += 1 @@ -664,88 +664,81 @@ def handle_delete(just_salvage=False): """ print("Getting list of builds in trash...") trashcan_tag = options.trashcan_tag - trash = sorted([(b['nvr'], b) for b in session.listTagged(trashcan_tag)]) - print("...got %i builds" % len(trash)) - # XXX - it would be better if there were more appropriate server calls for this grace_period = options.grace_period - import time + # using history makes for a smaller query that listTagged + history = session.queryHistory(tables=['tag_listing'], + tag=trashcan_tag, + active=True, + before=time.time()-grace_period) + # simulate needed binfo fields + binfos = history['tag_listing'] + for binfo in binfos: + binfo['nvr'] = '%(name)s-%(version)s-%(release)s' % binfo + binfo['id'] = binfo['build_id'] + to_check = {b['nvr']: b for b in binfos} + print(f"...got {len(to_check)} builds to check") - print("1st pass: blacklist") - continuing = [] - for nvr, binfo in trash: + print("1st pass: package filter") + for nvr in sorted(to_check): + binfo = to_check[nvr] if not check_package(binfo['name']): if options.debug: - print("Skipping package: %s" % nvr) - continue - continuing.append((nvr, binfo)) + print(f"Skipping package: {nvr}") + del to_check[nvr] + print(f"{len(to_check)} builds remaining") print("2nd pass: tags") - continuing, trash = [], continuing - mcall = koji.MultiCallSession(session, batch=100) - for nvr, binfo in trash: - mcall.listTags(build=binfo['id'], perms=False) - for (nvr, binfo), [tags] in zip(trash, mcall.call_all()): + with session.multicall(batch=100) as m: + tags = {nvr: m.listTags(build=binfo['id'], perms=False) for nvr in to_check} + for nvr in sorted(to_check): # see if build has been tagged elsewhere - tags = [t['name'] for t in tags if t['name'] != trashcan_tag] - if tags: - print("Build %s tagged elsewhere: %s" % (nvr, tags)) + binfo = to_check[nvr] + btags = [t['name'] for t in tags[nvr].result if t['name'] != trashcan_tag] + if btags: + print(f"Build {nvr} tagged elsewhere: {btags}") salvage_build(binfo) - continue - continuing.append((nvr, binfo)) + del to_check[nvr] + print(f"{len(to_check)} builds remaining") print("3rd pass: signatures") - continuing, trash = [], continuing - for nvr, binfo in trash: + for nvr in sorted(to_check): # check build signatures + binfo = to_check[nvr] keys = get_build_sigs(binfo['id'], cache=False) if keys and options.debug: - print("Build: %s, Keys: %s" % (nvr, keys)) + print(f"Build: {nvr}, Keys: {keys}") if protected_sig(keys): - print("Salvaging signed build %s. Keys: %s" % (nvr, keys)) + print(f"Salvaging signed build {nvr}. Keys: {keys}") salvage_build(binfo) - continue - if just_salvage: - # skip the rest when salvaging - continue - continuing.append((nvr, binfo)) + del to_check[nvr] + print(f"{len(to_check)} builds remaining") - print("4th pass: history") - continuing, trash = [], continuing - for nvr, binfo in trash: - # determine how long this build has been in the trashcan - mcall.queryHistory(build=binfo['id'], tag=trashcan_tag) + if just_salvage: + print("Salvage mode. Skipping deletes.") + return - for (nvr, binfo), [history] in zip(trash, mcall.call_all()): - current = [x for x in history['tag_listing'] if x['active']] - if not current: - # untagged just now? - print("Warning: history missing for %s" % nvr) - pprint.pprint(binfo) - pprint.pprint(history) - continue - assert len(current) == 1 # see db constraint - current = current[0] - age = time.time() - current['create_ts'] - if age < grace_period: - if options.debug: - print("Skipping build %s, age=%i" % (nvr, age)) - continue - continuing.append(binfo) - - print("5th pass: deletion") - for binfo in continuing: + print("4th pass: deletion") + if options.test: + for nvr in sorted(to_check): + binfo = to_check[nvr] + print(f"Would have deleted build from trashcan: {nvr}") + else: # go ahead and delete - if options.test: - print("Would have deleted build from trashcan: %s" % binfo['nvr']) - else: - print("Deleting build: %s" % binfo['nvr']) - mcall.untagBuildBypass(trashcan_tag, binfo['id']) - mcall.deleteBuild(binfo['id']) - - for binfo, result in zip(continuing, mcall.call_all()): - if isinstance(result, dict): - print("Warning: deletion failed: %s" % result['faultString']) - # TODO - log details for delete failures + with session.multicall(batch=100) as m: + untags = {nvr: m.untagBuildBypass(trashcan_tag, to_check[nvr]['id']) for nvr in to_check} + deletes = {nvr: m.deleteBuild(to_check[nvr]['id']) for nvr in to_check} + for nvr in sorted(to_check): + # check multicall results + try: + r = untags[nvr].result + except GenericError as e: + print(f"Failed to untag {nvr} from trashcan: {e}") + try: + r = deletes[nvr].result + except GenericError as e: + print(f"Warning: deletion failed for {nvr}: ({e})") + continue + print(f"Deleted build: {nvr}") class TagPruneTest(koji.policy.MatchTest):