particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

PR#3608: koji-gc: use history to query trashcan contents

Browse source 
Merges #3608
https://pagure.io/koji/pull-request/3608
This commit is contained in:
Tomas Kopecek 2022-12-07 15:30:24 +01:00
commit eba7cbd126
1 changed files with 58 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

123
util/koji-gc
View file

@ -483,7 +483,7 @@ def handle_trash():
N = len(untagged) N = len(untagged)
to_trash = [] to_trash = []
print("1st pass: blacklist") print("1st pass: package filter")
continuing = [] continuing = []
for binfo in untagged: for binfo in untagged:
i += 1 i += 1
@ -664,88 +664,81 @@ def handle_delete(just_salvage=False):
""" """
print("Getting list of builds in trash...") print("Getting list of builds in trash...")
trashcan_tag = options.trashcan_tag trashcan_tag = options.trashcan_tag
trash = sorted([(b['nvr'], b) for b in session.listTagged(trashcan_tag)])
print("...got %i builds" % len(trash))
# XXX - it would be better if there were more appropriate server calls for this
grace_period = options.grace_period grace_period = options.grace_period
import time # using history makes for a smaller query that listTagged
history = session.queryHistory(tables=['tag_listing'],
tag=trashcan_tag,
active=True,
before=time.time()-grace_period)
# simulate needed binfo fields
binfos = history['tag_listing']
for binfo in binfos:
binfo['nvr'] = '%(name)s-%(version)s-%(release)s' % binfo
binfo['id'] = binfo['build_id']
to_check = {b['nvr']: b for b in binfos}
print(f"...got {len(to_check)} builds to check")
print("1st pass: blacklist") print("1st pass: package filter")
continuing = [] for nvr in sorted(to_check):
for nvr, binfo in trash: binfo = to_check[nvr]
if not check_package(binfo['name']): if not check_package(binfo['name']):
if options.debug: if options.debug:
print("Skipping package: %s" % nvr) print(f"Skipping package: {nvr}")
continue del to_check[nvr]
continuing.append((nvr, binfo)) print(f"{len(to_check)} builds remaining")
print("2nd pass: tags") print("2nd pass: tags")
continuing, trash = [], continuing with session.multicall(batch=100) as m:
mcall = koji.MultiCallSession(session, batch=100) tags = {nvr: m.listTags(build=binfo['id'], perms=False) for nvr in to_check}
for nvr, binfo in trash: for nvr in sorted(to_check):
mcall.listTags(build=binfo['id'], perms=False)
for (nvr, binfo), [tags] in zip(trash, mcall.call_all()):
# see if build has been tagged elsewhere # see if build has been tagged elsewhere
tags = [t['name'] for t in tags if t['name'] != trashcan_tag] binfo = to_check[nvr]
if tags: btags = [t['name'] for t in tags[nvr].result if t['name'] != trashcan_tag]
print("Build %s tagged elsewhere: %s" % (nvr, tags)) if btags:
print(f"Build {nvr} tagged elsewhere: {btags}")
salvage_build(binfo) salvage_build(binfo)
continue del to_check[nvr]
continuing.append((nvr, binfo)) print(f"{len(to_check)} builds remaining")
print("3rd pass: signatures") print("3rd pass: signatures")
continuing, trash = [], continuing for nvr in sorted(to_check):
for nvr, binfo in trash:
# check build signatures # check build signatures
binfo = to_check[nvr]
keys = get_build_sigs(binfo['id'], cache=False) keys = get_build_sigs(binfo['id'], cache=False)
if keys and options.debug: if keys and options.debug:
print("Build: %s, Keys: %s" % (nvr, keys)) print(f"Build: {nvr}, Keys: {keys}")
if protected_sig(keys): if protected_sig(keys):
print("Salvaging signed build %s. Keys: %s" % (nvr, keys)) print(f"Salvaging signed build {nvr}. Keys: {keys}")
salvage_build(binfo) salvage_build(binfo)
continue del to_check[nvr]
if just_salvage: print(f"{len(to_check)} builds remaining")
# skip the rest when salvaging
continue
continuing.append((nvr, binfo))
print("4th pass: history") if just_salvage:
continuing, trash = [], continuing print("Salvage mode. Skipping deletes.")
for nvr, binfo in trash: return
# determine how long this build has been in the trashcan
mcall.queryHistory(build=binfo['id'], tag=trashcan_tag)
for (nvr, binfo), [history] in zip(trash, mcall.call_all()): print("4th pass: deletion")
current = [x for x in history['tag_listing'] if x['active']] if options.test:
if not current: for nvr in sorted(to_check):
# untagged just now? binfo = to_check[nvr]
print("Warning: history missing for %s" % nvr) print(f"Would have deleted build from trashcan: {nvr}")
pprint.pprint(binfo) else:
pprint.pprint(history)
continue
assert len(current) == 1 # see db constraint
current = current[0]
age = time.time() - current['create_ts']
if age < grace_period:
if options.debug:
print("Skipping build %s, age=%i" % (nvr, age))
continue
continuing.append(binfo)
print("5th pass: deletion")
for binfo in continuing:
# go ahead and delete # go ahead and delete
if options.test: with session.multicall(batch=100) as m:
print("Would have deleted build from trashcan: %s" % binfo['nvr']) untags = {nvr: m.untagBuildBypass(trashcan_tag, to_check[nvr]['id']) for nvr in to_check}
else: deletes = {nvr: m.deleteBuild(to_check[nvr]['id']) for nvr in to_check}
print("Deleting build: %s" % binfo['nvr']) for nvr in sorted(to_check):
mcall.untagBuildBypass(trashcan_tag, binfo['id']) # check multicall results
mcall.deleteBuild(binfo['id']) try:
r = untags[nvr].result
for binfo, result in zip(continuing, mcall.call_all()): except GenericError as e:
if isinstance(result, dict): print(f"Failed to untag {nvr} from trashcan: {e}")
print("Warning: deletion failed: %s" % result['faultString']) try:
# TODO - log details for delete failures r = deletes[nvr].result
except GenericError as e:
print(f"Warning: deletion failed for {nvr}: ({e})")
continue
print(f"Deleted build: {nvr}")
class TagPruneTest(koji.policy.MatchTest): class TagPruneTest(koji.policy.MatchTest):