f41b8c70a7
- putting xmlrpc stuff into koji.xmlrpcplus - adding koji.xmlrpcplus.xmlrpc_server to refer - replacing refs of original xmlrpc.client.dumps to enhanced koji.xmlrpcplus.dumps fixes: #3964
275 lines
11 KiB
Python
Executable file
275 lines
11 KiB
Python
Executable file
#!/usr/bin/python3
|
|
|
|
import os
|
|
from optparse import OptionParser
|
|
from koji.context import context
|
|
|
|
import koji
|
|
import kojihub.db
|
|
import koji.xmlrpcplus
|
|
from kojihub.db import DeleteProcessor, QueryProcessor, BulkInsertProcessor
|
|
|
|
|
|
def clean_sessions(cursor, vacuum, test, age, absolute):
|
|
clause = f"(update_time < NOW() - '{age:d} days'::interval)"
|
|
if absolute is not None:
|
|
clause += f"OR (start_time < NOW() - '{absolute:d} days'::interval)"
|
|
|
|
if options.verbose:
|
|
query = QueryProcessor(tables=['sessions'], clauses=[clause], opts={'countOnly': True})
|
|
rows = query.execute()
|
|
print(f"Deleting {rows} sessions")
|
|
|
|
if not test:
|
|
delete = DeleteProcessor(table='sessions', clauses=[clause])
|
|
delete.execute()
|
|
if vacuum:
|
|
cursor.execute("VACUUM ANALYZE sessions")
|
|
|
|
|
|
def clean_reservations(cursor, vacuum, test, age):
|
|
clauses = [f"created < NOW() - '{age} days'::interval"]
|
|
if options.verbose:
|
|
query = QueryProcessor(tables=['build_reservations'], clauses=clauses,
|
|
opts={'countOnly': True})
|
|
rows = query.execute()
|
|
print(f"Deleting {rows} build reservations")
|
|
|
|
if not test:
|
|
delete = DeleteProcessor(table='build_reservations', clauses=clauses)
|
|
delete.execute()
|
|
if vacuum:
|
|
cursor.execute("VACUUM ANALYZE build_reservations")
|
|
|
|
|
|
def clean_notification_tasks(cursor, vacuum, test, age):
|
|
clauses = ["method = 'tagNotification'",
|
|
f"completion_time < NOW() - '{age} days'::interval"]
|
|
if options.verbose:
|
|
query = QueryProcessor(tables=['task'], clauses=clauses, opts={'countOnly': True})
|
|
rows = query.execute()
|
|
print(f"Deleting {rows} tagNotification tasks")
|
|
|
|
if not test:
|
|
# cascades
|
|
delete = DeleteProcessor(table='task', clauses=clauses)
|
|
delete.execute()
|
|
if vacuum:
|
|
cursor.execute("VACUUM ANALYZE task")
|
|
|
|
|
|
def clean_scratch_tasks(cursor, vacuum, test, age):
|
|
clauses = ["method = 'build'",
|
|
f"completion_time < NOW() - '{age} days'::interval",
|
|
"request LIKE '%%<name>scratch</name>%%'"]
|
|
if options.verbose:
|
|
query = QueryProcessor(tables=['task'], clauses=clauses, opts={'countOnly': True})
|
|
rows = query.execute()
|
|
print(f"Deleting {rows} scratch build tasks")
|
|
|
|
if test:
|
|
return
|
|
|
|
# we should check, that it is really a scratch build
|
|
ids = []
|
|
# will be dropped automatically in the end of script/connection
|
|
cursor.execute("CREATE TEMPORARY TABLE temp_scratch_tasks (task_id INTEGER NOT NULL)")
|
|
query = QueryProcessor(tables=['task'], columns=['id', 'request'], clauses=clauses)
|
|
for row in query.execute():
|
|
task_id = row['id']
|
|
request = row['request']
|
|
try:
|
|
params, _ = koji.xmlrpcplus.xmlrpc_client.loads(request)
|
|
opts = params[2]
|
|
if opts['scratch']:
|
|
ids.append(task_id)
|
|
except Exception:
|
|
continue
|
|
insert = BulkInsertProcessor('temp_scratch_tasks')
|
|
for task_id in ids:
|
|
insert.add_record(task_id=task_id)
|
|
insert.execute()
|
|
|
|
parents = ids
|
|
while True:
|
|
if not parents:
|
|
break
|
|
children = []
|
|
query = QueryProcessor(tables=['task'],
|
|
columns=['id'],
|
|
clauses=["parent IN %(parents)s"],
|
|
values={'parents': parents})
|
|
for row in query.execute():
|
|
children.append(row['id'])
|
|
parents = children
|
|
if children:
|
|
insert = BulkInsertProcessor(table='temp_scratch_tasks')
|
|
for task_id in children:
|
|
insert.add_record(task_id=task_id)
|
|
insert.execute()
|
|
|
|
if not ids:
|
|
return
|
|
|
|
# delete from referring tables
|
|
ref_tables = (
|
|
'standard_buildroot',
|
|
'scheduler_task_runs',
|
|
'scheduler_task_refusals',
|
|
'scheduler_log_messages',
|
|
)
|
|
for table in ref_tables:
|
|
delete = DeleteProcessor(table=table,
|
|
clauses=['task_id IN (SELECT task_id FROM temp_scratch_tasks)'])
|
|
delete.execute()
|
|
|
|
# delete tasks finally
|
|
delete = DeleteProcessor(table='task',
|
|
clauses=['id IN (SELECT task_id FROM temp_scratch_tasks)'])
|
|
delete.execute()
|
|
|
|
if vacuum:
|
|
cursor.execute("VACUUM ANALYZE standard_buildroot")
|
|
cursor.execute("VACUUM ANALYZE task")
|
|
|
|
|
|
def clean_buildroots(cursor, vacuum, test):
|
|
if options.verbose:
|
|
clauses = ["cg_id IS NULL", "id NOT IN (SELECT buildroot_id FROM standard_buildroot)"]
|
|
query = QueryProcessor(tables=['buildroot'], clauses=clauses, opts={'countOnly': True})
|
|
rows = query.execute()
|
|
print(f"Deleting {rows} buildroots")
|
|
|
|
if not test:
|
|
q = "FROM buildroot WHERE cg_id IS NULL AND id NOT IN " \
|
|
"(SELECT buildroot_id FROM standard_buildroot)"
|
|
delete = DeleteProcessor(table='buildroot_listing',
|
|
clauses=[f'buildroot_id IN (SELECT id {q})'])
|
|
delete.execute()
|
|
clauses = ['cg_id IS NULL AND id NOT IN (SELECT buildroot_id FROM standard_buildroot)']
|
|
delete = DeleteProcessor(table='buildroot', clauses=clauses)
|
|
delete.execute()
|
|
if vacuum:
|
|
cursor.execute("VACUUM ANALYZE buildroot_listing")
|
|
cursor.execute("VACUUM ANALYZE buildroot")
|
|
|
|
|
|
def clean_scheduler_logs(cursor, vacuum, test, age):
|
|
clauses = [f"(msg_time < NOW() - '{age:d} days'::interval)"]
|
|
if options.verbose:
|
|
query = QueryProcessor(tables=["scheduler_log_messages"],
|
|
clauses=clauses,
|
|
opts={'countOnly': True})
|
|
rows = query.execute()
|
|
print(f"Deleting {rows} scheduler log messages")
|
|
if not test:
|
|
delete = DeleteProcessor(table="scheduler_log_messages", clauses=clauses)
|
|
delete.execute()
|
|
if vacuum:
|
|
cursor.execute("VACUUM ANALYZE scheduler_log_messages")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
global options
|
|
parser = OptionParser("%prog cleans koji database")
|
|
parser.add_option('-v', '--verbose', action="store_true", help="Be verbose")
|
|
parser.add_option('-t', '--test', action="store_true",
|
|
help="Don't delete anything, print estimations (implies -v)")
|
|
parser.add_option('-c', '--conf', default='/etc/koji-hub/hub.conf',
|
|
action='store', help="Path to koji's hub.conf")
|
|
parser.add_option('--config-dir', default='/etc/koji-hub/hub.conf.d',
|
|
action='store', help="Path to koji's hub.conf directory")
|
|
parser.add_option('--no-vacuum', action="store_false", dest="vacuum",
|
|
default=True,
|
|
help="Don't run vacuum on affected tables")
|
|
parser.add_option('--sessions-age', type=int,
|
|
action="store", default=1, metavar="DAYS",
|
|
help="Delete inactive sessions older than this (default: 1 day)")
|
|
parser.add_option('--sessions-absolute-age', type=int,
|
|
action="store", default=None, metavar="DAYS",
|
|
help="Delete all sessions older than this (default: None)")
|
|
parser.add_option('--reservations-age', type=int,
|
|
action="store", default=1, metavar="DAYS",
|
|
help="Delete CG reservations older than this (default: 1 day)")
|
|
parser.add_option('--tag-notifications', action="store_true",
|
|
help="Delete tagNotification tasks", default=False)
|
|
parser.add_option('--tag-notifications-age', type=int,
|
|
action="store", default=730, metavar="DAYS",
|
|
help="Delete tagNotification tasks older than this (default: 2 years)")
|
|
parser.add_option("--scratch-builds", action="store_true",
|
|
dest="scratch", default=False,
|
|
help="Delete scratch build tasks")
|
|
parser.add_option('--scratch-builds-age', type=int, dest="scratch_age",
|
|
action="store", default=730, metavar="DAYS",
|
|
help="Delete scratch builds' tasks older than this (default: 2 years")
|
|
parser.add_option('--logs-age', type=int,
|
|
action="store", default=7, metavar="DAYS",
|
|
help="Delete scheduler log messages older than this (default: 7 days)")
|
|
parser.add_option('--buildroots', action="store_true",
|
|
help="Delete unreferenced buildroots")
|
|
parser.add_option('-f', '--force', action="store_true",
|
|
help="This is needs for all options affecting auditable data")
|
|
options, args = parser.parse_args()
|
|
|
|
if options.test:
|
|
options.verbose = True
|
|
print("Running in test mode, no changes will be made")
|
|
|
|
if not options.force and (options.tag_notifications or options.scratch or options.buildroots):
|
|
parser.error("You need to pass --force to trigger these actions")
|
|
|
|
if not os.path.exists(options.conf):
|
|
parser.error("Config file doesn't exist")
|
|
|
|
config = koji.read_config_files([options.config_dir, (options.conf, True)], raw=True)
|
|
|
|
cfgmap = [
|
|
# option, type, default
|
|
['DBName', 'string', None],
|
|
['DBUser', 'string', None],
|
|
['DBHost', 'string', None],
|
|
['DBhost', 'string', None], # alias for backwards compatibility
|
|
['DBPort', 'integer', None],
|
|
['DBPass', 'string', None],
|
|
['DBConnectionString', 'string', None],
|
|
]
|
|
|
|
opts = {}
|
|
for name, dtype, default in cfgmap:
|
|
key = ('hub', name)
|
|
if config and config.has_option(*key):
|
|
if dtype == 'integer':
|
|
opts[name] = config.getint(*key)
|
|
elif dtype == 'boolean':
|
|
opts[name] = config.getboolean(*key)
|
|
else:
|
|
opts[name] = config.get(*key)
|
|
continue
|
|
opts[name] = default
|
|
|
|
if opts.get('DBConnectionString'):
|
|
kojihub.db.provideDBopts(dsn=opts['DBConnectionString'])
|
|
else:
|
|
if opts['DBHost'] is None:
|
|
opts['DBHost'] = opts['DBhost']
|
|
kojihub.db.provideDBopts(database=opts["DBName"],
|
|
user=opts["DBUser"],
|
|
password=opts.get("DBPass", None),
|
|
host=opts.get("DBHost", None),
|
|
port=opts.get("DBPort", None))
|
|
|
|
context.cnx = kojihub.db.connect()
|
|
context.cnx.set_session(autocommit=True)
|
|
cursor = context.cnx.cursor()
|
|
|
|
clean_sessions(cursor, options.vacuum, options.test, options.sessions_age,
|
|
options.sessions_absolute_age)
|
|
clean_reservations(cursor, options.vacuum, options.test, options.reservations_age)
|
|
clean_scheduler_logs(cursor, options.vacuum, options.test, options.logs_age)
|
|
if options.tag_notifications:
|
|
clean_notification_tasks(cursor, options.vacuum, options.test,
|
|
age=options.tag_notifications_age)
|
|
if options.scratch:
|
|
clean_scratch_tasks(cursor, options.vacuum, options.test, age=options.scratch_age)
|
|
if options.buildroots:
|
|
clean_buildroots(cursor, options.vacuum, options.test)
|