PR#2485: builder: configurable TTL for buildroots

Merges #2485
https://pagure.io/koji/pull-request/2485

Fixes: #2374
https://pagure.io/koji/issue/2374
TaskManager: make some (max/min) time configurable in `updateBuildroots`

builder/kojid

@@ -6441,6 +6441,8 @@ def get_options():
             quit('invalid section found in config file: %s' % x)
     defaults = {'sleeptime': 15,
                 'maxjobs': 10,
+                'buildroot_basic_cleanup_delay': 120,
+                'buildroot_final_cleanup_delay': 86400,
                 'literal_task_arches': '',
                 'minspace': 8192,
                 'admin_emails': None,
@@ -6500,7 +6502,8 @@ def get_options():
             if name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval',
                         'max_retries', 'offline_retry_interval', 'failed_buildroot_lifetime',
                         'timeout', 'rpmbuild_timeout', 'oz_install_timeout',
-                        'task_avail_delay']:
+                        'task_avail_delay', 'buildroot_basic_cleanup_delay',
+                        'buildroot_final_cleanup_delay']:
                 try:
                     defaults[name] = int(value)
                 except ValueError:

builder/kojid.conf

@@ -5,6 +5,14 @@
 ; The maximum number of jobs that kojid will handle at a time
 ; maxjobs=10
 
+; Time after successfully finished task's buildroot is deleted (2 minutes in seconds)
+; Some logs and directories are left in place until buildroot_final_cleanup_delay
+; buildroot_basic_cleanup_delay=120
+
+
+; Time after successfully finished task's buildroot is deleted completely (1 day in seconds)
+; buildroot_final_cleanup_delay=86400
+
 ; The minimum amount of free space (in MBs) required for each build root
 ; minspace=8192
 

docs/source/kojid_conf.rst

@@ -25,6 +25,16 @@ General
       serves as a backup to the capacity check and prevents a tremendous
       number of low weight jobs from piling up.
 
+   buildroot_basic_cleanup_delay=120
+      Time after which successfully finished task's buildroot is deleted (2
+      minutes in seconds). Some logs and directories are left in place until
+      buildroot_final_cleanup_delay
+
+   buildroot_final_cleanup_delay=86400
+      Time after which buildroot (pre-cleand after
+      ``buildroot_basic_cleanup_delay``) is deleted completely. (1 day in
+      seconds)
+
    max_retries=120
       Set the maximum number of times that an individual hub call can be
       retried.

koji/daemon.py

@@ -737,7 +737,7 @@ class TaskManager(object):
                 # can lead to a world of hurt.
                 # We remove the rootdir contents but leave the rootdir unless it
                 # is really old
-                if age > 3600 * 24:
+                if age > self.options.buildroot_final_cleanup_delay:
                     # dir untouched for a day
                     self.logger.info("Removing buildroot: %s" % desc)
                     if ((topdir and safe_rmtree(topdir, unmount=True, strict=False) != 0) or
@@ -749,7 +749,7 @@ class TaskManager(object):
                         os.unlink(data['cfg'])
                     except OSError as e:
                         self.logger.warning("%s: can't remove config: %s" % (desc, e))
-                elif age > 120 and rootdir:
+                elif age > self.options.buildroot_basic_cleanup_delay and rootdir:
                     for d in (topdir, topdir_bootstrap):
                         if not d:
                             continue