From 92a773a19d17f05712fdb181948b5d2438f59463 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Tue, 6 Sep 2022 10:56:29 +0200 Subject: [PATCH] fix koji-sweep-db --- koji/db.py | 3 -- util/koji-sweep-db | 91 ++++++++++++++++++++-------------------------- 2 files changed, 40 insertions(+), 54 deletions(-) diff --git a/koji/db.py b/koji/db.py index 0a3a31f9..6e0b5497 100644 --- a/koji/db.py +++ b/koji/db.py @@ -226,9 +226,6 @@ def _dml(operation, values, log_errors=True): def _fetchMulti(query, values): """Run the query and return all rows""" - print('===================================') - print(context) - print('===================================') c = context.cnx.cursor() c.execute(query, values) results = c.fetchall() diff --git a/util/koji-sweep-db b/util/koji-sweep-db index 0d36c277..f0bba678 100755 --- a/util/koji-sweep-db +++ b/util/koji-sweep-db @@ -5,68 +5,64 @@ import xmlrpc.client from optparse import OptionParser from koji.context import context +import koji import koji.db +from koji.db import QueryProcessor, InsertProcessor, BulkInsertProcessor def clean_sessions(cursor, vacuum, test, age): - table = 'sessions' - clauses = [f"update_time < NOW() - '{int(age)} days'::interval"] + clauses = [f"update_time < NOW() - '{age} days'::interval"] if options.verbose: - query = koji.db.QueryProcessor(clauses=clauses, opts={'countOnly': True}, - tables=[table], values=locals()) - rows = query.iterate() + query = QueryProcessor(tables=['sessions'], clauses=clauses, opts={'countOnly': True}) + rows = query.execute() print(f"Deleting {rows} sessions") if not test: cursor.execute( - f"DELETE FROM sessions WHERE update_time < NOW() - '{int(age)} days'::interval") + f"DELETE FROM sessions WHERE update_time < NOW() - '{age} days'::interval") if vacuum: cursor.execute("VACUUM ANALYZE sessions") def clean_reservations(cursor, vacuum, test, age): - table = 'build_reservations' - clauses = [f"created < NOW() - '{int(age)} days'::interval"] if options.verbose: - query = koji.db.QueryProcessor(clauses=clauses, opts={'countOnly': True}, - tables=[table], values=locals()) - rows = query.iterate() + query = QueryProcessor( + tables=['build_reservations'], + clauses=[f"created < NOW() - '{age} days'::interval"], + opts={'countOnly': True}) + rows = query.execute() print(f"Deleting {rows} build reservations") if not test: cursor.execute( - f"DELETE FROM build_reservations WHERE created < NOW() - '{int(age)} days'::interval") + f"DELETE FROM build_reservations WHERE created < NOW() - '{age} days'::interval") if vacuum: cursor.execute("VACUUM ANALYZE build_reservations") def clean_notification_tasks(cursor, vacuum, test, age): - table = 'task' clauses = ["method = 'tagNotification'", - f"completion_time < NOW() - '{int(age)} days'::interval"] + f"completion_time < NOW() - '{age} days'::interval"] if options.verbose: - query = koji.db.QueryProcessor(clauses=clauses, opts={'countOnly': True}, - tables=[table], values=locals()) - rows = query.iterate() + query = QueryProcessor(tables=['task'], clauses=clauses, opts={'countOnly': True}) + rows = query.execute() print(f"Deleting {rows} tagNotification tasks") if not test: # cascade - cursor.execute(f"DELETE FROM task WHERE method = 'tagNotification' AND " - f"completion_time < NOW() - '{int(age)} days'::interval") + cursor.execute("DELETE FROM task WHERE method = 'tagNotification' AND " + f"completion_time < NOW() - '{age} days'::interval") if vacuum: cursor.execute("VACUUM ANALYZE task") def clean_scratch_tasks(cursor, vacuum, test, age): - table = 'task' clauses = ["method = 'build'", - f"completion_time < NOW() - '{int(age)} days'::interval", - "request LIKE '%%%%scratch%%%%'"] + f"completion_time < NOW() - '{age} days'::interval", + "request LIKE '%%scratch%%'"] if options.verbose: - query = koji.db.QueryProcessor(clauses=clauses, opts={'countOnly': True}, - tables=[table], values=locals()) - rows = query.iterate() + query = QueryProcessor(tables=['task'], clauses=clauses, opts={'countOnly': True}) + rows = query.execute() print(f"Deleting {rows} scratch build tasks") if test: @@ -76,42 +72,37 @@ def clean_scratch_tasks(cursor, vacuum, test, age): ids = [] # will be dropped automatically in the end of script/connection cursor.execute("CREATE TEMPORARY TABLE temp_scratch_tasks (task_id INTEGER NOT NULL)") - query = koji.db.QueryProcessor(columns=['id', 'request'], clauses=clauses, - tables=[table], values=locals()) - rows = query.execute() - for row in rows: + query = QueryProcessor(tables=['task'], columns=['id', 'request'], clauses=clauses) + for row in query.execute(): task_id, request = row try: - params, method = xmlrpc.client.loads(request) + params, _ = xmlrpc.client.loads(request) opts = params[2] if opts['scratch']: - insert = koji.db.InsertProcessor('temp_scratch_tasks') - insert.set((task_id,)) - insert.make_create() - insert.execute() ids.append(task_id) except Exception: continue + insert = BulkInsertProcessor('temp_scratch_tasks') + for task_id in ids: + insert.add_record(task_id=task_id) + insert.execute() parents = ids while True: if not parents: break children = [] - string_parents = ', '.join(parents) - query = koji.db.QueryProcessor(columns=['id'], - clauses=[f"parent IN ({int(age)})"], - tables=['task'], - values=locals()) - rows = query.execute() - for row in rows: + query = QueryProcessor(tables=['task'], + columns=['id'], + clauses=["parent IN %(parents)s"], + values={'parents': parents}) + for row in query.execute(): children.append(row[0]) parents = children if children: - values = ', '.join(["(%d)" % task_id for task_id in children]) - insert = koji.db.InsertProcessor('temp_scratch_tasks') - insert.set(values) - insert.make_create() + insert = BulkInsertProcessor(table='temp_scratch_tasks') + for task_id in children: + insert.add_record(task_id=task_id) insert.execute() if not ids: @@ -131,15 +122,13 @@ def clean_scratch_tasks(cursor, vacuum, test, age): def clean_buildroots(cursor, vacuum, test): if options.verbose: - clauses = ["cg_id IS NULL", - "id NOT IN (SELECT buildroot_id FROM standard_buildroot)"] - query = koji.db.QueryProcessor(clauses=clauses, opts={'countOnly': True}, - tables=['buildroot'], values=locals()) - rows = query.iterate() + clauses = ["cg_id IS NULL", "id NOT IN (SELECT buildroot_id FROM standard_buildroot)"] + query = QueryProcessor(tables=['buildroot'], clauses=clauses, opts={'countOnly': True}) + rows = query.execute() print(f"Deleting {rows} buildroots") if not test: - q = " FROM buildroot WHERE cg_id IS NULL AND id NOT IN " \ + q = "FROM buildroot WHERE cg_id IS NULL AND id NOT IN " \ "(SELECT buildroot_id FROM standard_buildroot)" cursor.execute(f"DELETE FROM buildroot_listing WHERE buildroot_id IN (SELECT id {q})") cursor.execute(f"DELETE {q}")