From d412fc54d79f4dc0092d2d4fb3e1a8b754e46038 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Fri, 7 Mar 2008 14:36:23 -0500 Subject: [PATCH] allow creation of repos from a specified event --- builder/kojid | 12 +++++++++--- hub/kojihub.py | 34 +++++++++++++++++++++++++--------- util/kojira | 27 +++++++++++++++++++++------ 3 files changed, 55 insertions(+), 18 deletions(-) diff --git a/builder/kojid b/builder/kojid index 991eebae..3913305d 100755 --- a/builder/kojid +++ b/builder/kojid @@ -2228,10 +2228,13 @@ class NewRepoTask(BaseTaskHandler): Methods = ['newRepo'] _taskWeight = 0.1 - def handler(self, tag): + def handler(self, tag, event=None): self.uploadpath = self.getUploadDir() tinfo = session.getTag(tag, strict=True) - repo_id, event_id = session.host.repoInit(tinfo['id']) + kwargs = {} + if event is not None: + kwargs['event'] = event + repo_id, event_id = session.host.repoInit(tinfo['id'], **kwargs) path = koji.pathinfo.repo(repo_id, tinfo['name']) if not os.path.isdir(path): raise koji.GenericError, "Repo directory missing: %s" % path @@ -2254,7 +2257,10 @@ class NewRepoTask(BaseTaskHandler): for (arch, task_id) in subtasks.iteritems(): data[arch] = results[task_id] self.logger.debug("DEBUG: %r : %r " % (arch,data[arch],)) - session.host.repoDone(repo_id, data) + kwargs = {} + if event is not None: + kwargs['expire'] = True + session.host.repoDone(repo_id, data, **kwargs) return repo_id, event_id class CreaterepoTask(BaseTaskHandler): diff --git a/hub/kojihub.py b/hub/kojihub.py index 458e6bef..b87e2ec6 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -1679,7 +1679,7 @@ def get_task_descendents(task, childMap=None, request=False): get_task_descendents(Task(child['id']), childMap, request) return childMap -def repo_init(tag, with_src=False, with_debuginfo=False): +def repo_init(tag, with_src=False, with_debuginfo=False, event=None): """Create a new repo entry in the INIT state, return full repo data Returns a dictionary containing @@ -1694,7 +1694,13 @@ def repo_init(tag, with_src=False, with_debuginfo=False): for arch in tinfo['arches'].split(): repo_arches[koji.canonArch(arch)] = 1 repo_id = _singleValue("SELECT nextval('repo_id_seq')") - event_id = _singleValue("SELECT get_event()") + if event is None: + event_id = _singleValue("SELECT get_event()") + else: + #make sure event is valid + q = "SELECT time FROM events WHERE id=%(event)s" + event_time = _singleValue(q, locals(), strict=True) + event_id = event q = """INSERT INTO repo(id, create_event, tag_id, state) VALUES(%(repo_id)s, %(event_id)s, %(tag_id)s, %(state)s)""" _dml(q,locals()) @@ -4916,10 +4922,14 @@ class RootExports(object): repoInfo = staticmethod(repo_info) getActiveRepos = staticmethod(get_active_repos) - def newRepo(self, tag): + def newRepo(self, tag, event=None): """Create a newRepo task. returns task id""" context.session.assertPerm('repo') - return make_task('newRepo', [tag], priority=15, channel='createrepo') + if event: + args = koji.encode_args(tag, event=None) + else: + args = [tag] + return make_task('newRepo', args, priority=15, channel='createrepo') def repoExpire(self, repo_id): """mark repo expired""" @@ -6173,11 +6183,11 @@ class HostExports(object): br.assertTask(task_id) return br.updateList(rpmlist) - def repoInit(self, tag, with_src=False): + def repoInit(self, tag, with_src=False, event=None): """Initialize a new repo for tag""" host = Host() host.verify() - return repo_init(tag, with_src=with_src) + return repo_init(tag, with_src=with_src, event=event) def repoAddRPM(self, repo_id, path): """Add an uploaded rpm to a repo""" @@ -6222,11 +6232,14 @@ class HostExports(object): else: os.link(filepath, dst) - def repoDone(self, repo_id, data): + def repoDone(self, repo_id, data, expire=False): """Move repo data into place, mark as ready, and expire earlier repos repo_id: the id of the repo data: a dictionary of the form { arch: (uploadpath, files), ...} + expire(optional): if set to true, mark the repo expired immediately* + + * This is used when a repo from an older event is generated """ host = Host() host.verify() @@ -6248,8 +6261,11 @@ class HostExports(object): raise koji.GenericError, "uploaded file missing: %s" % src os.link(src, dst) os.unlink(src) - repo_ready(repo_id) - repo_expire_older(rinfo['tag_id'], rinfo['create_event']) + if expire: + repo_expire(repo_id) + else: + repo_ready(repo_id) + repo_expire_older(rinfo['tag_id'], rinfo['create_event']) def isEnabled(self): host = Host() diff --git a/util/kojira b/util/kojira index 0cce4a1d..431b4f85 100755 --- a/util/kojira +++ b/util/kojira @@ -119,8 +119,10 @@ class ManagedRepo(object): def tryDelete(self): """Remove the repo from disk, if possible""" + #we check just the event age first since it is faster age = time.time() - self.event_ts if age < options.deleted_repo_lifetime: + #XXX should really be called expired_repo_lifetime return False self.logger.debug("Attempting to delete repo %s.." % self.repo_id) if self.state != koji.REPO_EXPIRED: @@ -138,6 +140,16 @@ class ManagedRepo(object): return False tag_name = tag_info['name'] path = pathinfo.repo(self.repo_id, tag_name) + #also check dir age. We do this because a repo can be created from an older event + #and should not be removed based solely on that event's timestamp. + try: + age = time.time() - os.stat(path).st_mtime + except OSError: + self.logger.error("Can't stat repo directory: %s" % path) + return True + if age < options.deleted_repo_lifetime: + #XXX should really be called expired_repo_lifetime + return False safe_rmtree(path, strict=False) return True @@ -227,13 +239,14 @@ class RepoManager(object): if self.repos.has_key(repo_id): #we're already managing it, no need to deal with it here continue + try: + dir_ts = os.stat(repodir).st_mtime + except OSError: + #just in case something deletes the repo out from under us + continue rinfo = session.repoInfo(repo_id) if rinfo is None: - try: - age = time.time() - os.stat(repodir).st_mtime - except OSError: - #just in case something deletes the repo out from under us - continue + age = time.time() - dir_ts if age > 36000: if not options.ignore_stray_repos: self.logger.warn("Unexpected directory (no such repo): %s" % repodir) @@ -242,8 +255,9 @@ class RepoManager(object): self.logger.warn("Tag name mismatch: %s" % repodir) continue if rinfo['state'] in (koji.REPO_DELETED, koji.REPO_PROBLEM): - age = time.time() - rinfo['create_ts'] + age = time.time() - max(rinfo['create_ts'], dir_ts) if age > options.deleted_repo_lifetime: + #XXX should really be called expired_repo_lifetime count += 1 logger.info("Removing stray repo (state=%s): %s" % (koji.REPO_STATES[rinfo['state']], repodir)) safe_rmtree(repodir, strict=False) @@ -432,6 +446,7 @@ def get_options(): 'delete_batch_size': 3, 'max_repo_tasks' : 10, 'deleted_repo_lifetime': 7*24*3600, + #XXX should really be called expired_repo_lifetime 'cert': '/etc/kojira/client.crt', 'ca': '/etc/kojira/clientca.crt', 'serverca': '/etc/kojira/serverca.crt'