From 4a96ee5474552d41de37ca45e54d0869f366232b Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Thu, 2 Nov 2017 15:51:32 +0100 Subject: [PATCH 1/7] store git commit hash to build.source Git HEAD commit hash is now stored in build.info field. this field is currently used by content generators, but not by other builds. Also original checked out url is stored in build.extra.url field. Fixes: https://pagure.io/koji/issue/550 --- builder/kojid | 6 ++++++ koji/daemon.py | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/builder/kojid b/builder/kojid index e5f71964..0953dbfb 100755 --- a/builder/kojid +++ b/builder/kojid @@ -868,6 +868,10 @@ class BuildTask(BaseTaskHandler): h = self.readSRPMHeader(srpm) data = koji.get_header_fields(h,['name','version','release','epoch']) data['task_id'] = self.id + if getattr(self, 'source'): + data['source'] = self.source['source'] + data['extra'] = {'url': self.source['url']} + extra_arches = None self.logger.info("Reading package config for %(name)s" % data) pkg_cfg = self.session.getPackageConfig(dest_tag,data['name'],event=self.event_id) @@ -933,6 +937,7 @@ class BuildTask(BaseTaskHandler): parent=self.id) # wait for subtask to finish result = self.wait(task_id)[task_id] + self.source = result['source'] srpm = result['srpm'] return srpm @@ -4565,6 +4570,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask): 'logs': ["%s/%s" % (uploadpath, os.path.basename(f)) for f in log_files], 'brootid': brootid, + 'source': scm.get_source(), } class TagNotificationTask(BaseTaskHandler): diff --git a/koji/daemon.py b/koji/daemon.py index f279a67f..1153b161 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -32,6 +32,7 @@ import urlparse from fnmatch import fnmatch import base64 import time +import subprocess import sys import traceback import errno @@ -501,9 +502,28 @@ class SCM(object): rel_path = '../' * len(path_comps.split('/')) os.symlink(rel_path + 'common', '%s/../common' % sourcedir) + self.sourcedir = sourcedir return sourcedir ## END kojikamid dup + def get_source(self): + r = { + 'url': self.url, + 'source': '', + } + if self.scmtype.startswith('GIT'): + cmd = ['git', 'rev-parse', 'HEAD'] + fragment = subprocess.check_output(cmd, cwd=self.sourcedir).strip() + scheme = self.scheme[:-3] + if self.user: + netloc = '%s@%s' % (self.user, self.host) + else: + netloc = self.host + path = self.repository + query = self.module + r['source'] = urlparse.urlunsplit([scheme, netloc, path, query, fragment]) + return r + class TaskManager(object): From 3bad1f9f646f8eaa218844a07a33292b7a0f02f6 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mon, 6 Nov 2017 11:15:41 +0100 Subject: [PATCH 2/7] fix race condition and missing values --- builder/kojid | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/builder/kojid b/builder/kojid index 0953dbfb..4f60a8d9 100755 --- a/builder/kojid +++ b/builder/kojid @@ -868,7 +868,7 @@ class BuildTask(BaseTaskHandler): h = self.readSRPMHeader(srpm) data = koji.get_header_fields(h,['name','version','release','epoch']) data['task_id'] = self.id - if getattr(self, 'source'): + if getattr(self, 'source', False): data['source'] = self.source['source'] data['extra'] = {'url': self.source['url']} @@ -4564,13 +4564,14 @@ class BuildSRPMFromSCMTask(BaseBuildTask): brootid = broot.id log_files = glob.glob('%s/*.log' % broot.resultdir()) + source = scm.get_source() broot.expire() return {'srpm': "%s/%s" % (uploadpath, srpm_name), 'logs': ["%s/%s" % (uploadpath, os.path.basename(f)) for f in log_files], 'brootid': brootid, - 'source': scm.get_source(), + 'source': source, } class TagNotificationTask(BaseTaskHandler): From 0ae68fbbcb9fb0fad4f8fe747ad5f43fa4930555 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Mon, 6 Nov 2017 11:46:34 +0100 Subject: [PATCH 3/7] store source also for CVS/SVN --- koji/daemon.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/koji/daemon.py b/koji/daemon.py index 1153b161..c058624b 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -511,21 +511,23 @@ class SCM(object): 'url': self.url, 'source': '', } - if self.scmtype.startswith('GIT'): + if self.scmtype.startswith('CVS') or self.scmtype.startswith('SVN'): + scheme = self.scheme[:-3] + netloc = self.host + path = self.repository + query = self.module + fragment = self.revision + elif self.scmtype.startswith('GIT'): cmd = ['git', 'rev-parse', 'HEAD'] fragment = subprocess.check_output(cmd, cwd=self.sourcedir).strip() scheme = self.scheme[:-3] - if self.user: - netloc = '%s@%s' % (self.user, self.host) - else: - netloc = self.host + netloc = self.host path = self.repository query = self.module r['source'] = urlparse.urlunsplit([scheme, netloc, path, query, fragment]) return r - class TaskManager(object): def __init__(self, options, session): From 99dcb3798df09329d30b788107258ab5a7501bef Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 7 Dec 2017 16:10:13 -0500 Subject: [PATCH 4/7] avoid splitting SCM class in kojikamid --- koji/daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koji/daemon.py b/koji/daemon.py index c058624b..78c0c4d2 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -505,7 +505,6 @@ class SCM(object): self.sourcedir = sourcedir return sourcedir -## END kojikamid dup def get_source(self): r = { 'url': self.url, @@ -526,6 +525,7 @@ class SCM(object): query = self.module r['source'] = urlparse.urlunsplit([scheme, netloc, path, query, fragment]) return r +## END kojikamid dup class TaskManager(object): From d0c4f29b444405405ef142022e061649b2d126ed Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 7 Dec 2017 16:15:24 -0500 Subject: [PATCH 5/7] just use original url for non-git sources --- koji/daemon.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/koji/daemon.py b/koji/daemon.py index 78c0c4d2..4e6a9740 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -510,13 +510,7 @@ class SCM(object): 'url': self.url, 'source': '', } - if self.scmtype.startswith('CVS') or self.scmtype.startswith('SVN'): - scheme = self.scheme[:-3] - netloc = self.host - path = self.repository - query = self.module - fragment = self.revision - elif self.scmtype.startswith('GIT'): + if self.scmtype.startswith('GIT'): cmd = ['git', 'rev-parse', 'HEAD'] fragment = subprocess.check_output(cmd, cwd=self.sourcedir).strip() scheme = self.scheme[:-3] @@ -524,6 +518,9 @@ class SCM(object): path = self.repository query = self.module r['source'] = urlparse.urlunsplit([scheme, netloc, path, query, fragment]) + else: + # just use the same url + r['source'] = self.url return r ## END kojikamid dup From 15709e019c1b5c431121335d538588776b4d98b7 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 7 Dec 2017 16:59:09 -0500 Subject: [PATCH 6/7] adjust how we store the extra source data --- builder/kojid | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/kojid b/builder/kojid index 4f60a8d9..aba9bdda 100755 --- a/builder/kojid +++ b/builder/kojid @@ -870,7 +870,7 @@ class BuildTask(BaseTaskHandler): data['task_id'] = self.id if getattr(self, 'source', False): data['source'] = self.source['source'] - data['extra'] = {'url': self.source['url']} + data['extra'] = {'source': {'original_url': self.source['url']}} extra_arches = None self.logger.info("Reading package config for %(name)s" % data) From fab1cc4245151caa7d5c936080dd34dcf3393520 Mon Sep 17 00:00:00 2001 From: Mike McLean Date: Thu, 7 Dec 2017 17:16:15 -0500 Subject: [PATCH 7/7] don't fail if child task doesn't provide source While the current code will always provide it, the older code did not. When a Koji system is updated to this version, we could have a restarted build task read the result of a previously finished buildSRPMFromSCM task that ran with the old code. --- builder/kojid | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/builder/kojid b/builder/kojid index aba9bdda..a71de546 100755 --- a/builder/kojid +++ b/builder/kojid @@ -937,7 +937,10 @@ class BuildTask(BaseTaskHandler): parent=self.id) # wait for subtask to finish result = self.wait(task_id)[task_id] - self.source = result['source'] + if 'source' in result: + self.source = result['source'] + else: + self.logger.warning('subtask did not provide source data') srpm = result['srpm'] return srpm