From 64df809ab6a6d486bfb3db75e2bb75ae828a0733 Mon Sep 17 00:00:00 2001 From: Mike Bonnet Date: Fri, 17 Sep 2010 11:27:33 -0400 Subject: [PATCH] add support for wrapping win-build output in rpms --- builder/kojid | 126 +++++++++++++++++--------------- cli/koji | 4 +- docs/schema-upgrade-1.4-1.5.sql | 1 + docs/schema.sql | 1 + hub/kojihub.py | 38 +++++++--- koji/__init__.py | 5 +- koji/tasks.py | 16 ++++ vm/kojivmd | 53 +++++++++----- 8 files changed, 155 insertions(+), 89 deletions(-) diff --git a/builder/kojid b/builder/kojid index 6f334d79..05f806d3 100755 --- a/builder/kojid +++ b/builder/kojid @@ -31,7 +31,7 @@ import glob import logging import logging.handlers from koji.daemon import incremental_upload, log_output, TaskManager, SCM -from koji.tasks import ServerExit, BaseTaskHandler +from koji.tasks import ServerExit, BaseTaskHandler, MultiPlatformTask from koji.util import parseStatus, isSuccess import os import pwd @@ -1014,7 +1014,7 @@ class BuildArchTask(BaseTaskHandler): return ret -class MavenTask(BaseTaskHandler): +class MavenTask(MultiPlatformTask): Methods = ['maven'] @@ -1072,13 +1072,14 @@ class MavenTask(BaseTaskHandler): raise koji.BuildError, "package %s is blocked for tag %s" \ % (build_info['name'], dest_tag['name']) - self.build_id, build_info = self.session.host.initMavenBuild(self.id, build_info, maven_info) + build_info = self.session.host.initMavenBuild(self.id, build_info, maven_info) + self.build_id = build_info['id'] try: rpm_results = None spec_url = self.opts.get('specfile') if spec_url: - rpm_results = self.buildWrapperRPM(spec_url, build_tag, build_info, repo_id) + rpm_results = self.buildWrapperRPM(spec_url, self.build_task_id, build_tag, build_info, repo_id) if not self.opts.get('scratch'): self.session.host.completeMavenBuild(self.id, self.build_id, maven_results, rpm_results) @@ -1100,20 +1101,6 @@ class MavenTask(BaseTaskHandler): arch='noarch') self.wait(tag_task_id) - def buildWrapperRPM(self, spec_url, build_tag, build, repo_id): - task = self.session.getTaskInfo(self.build_task_id) - arglist = [spec_url, build_tag, build, task, {'repo_id': repo_id}] - - rpm_task_id = self.session.host.subtask(method='wrapperRPM', - arglist=arglist, - label='rpm', - parent=self.id, - arch='noarch') - results = self.wait(rpm_task_id)[rpm_task_id] - results['task_id'] = rpm_task_id - - return results - class BuildMavenTask(BaseTaskHandler): Methods = ['buildMaven'] @@ -1318,22 +1305,32 @@ class WrapperRPMTask(BaseTaskHandler): if not opts: opts = {} + if not (build or task): + raise koji.BuildError, 'build and/or task must be specified' + values = {} - maven_info = None + if build: + maven_info = self.session.getMavenBuild(build['id'], strict=False) + win_info = self.session.getWinBuild(build['id'], strict=False) + else: + maven_info = None + win_info = None + # list of artifact paths relative to kojiroot (not exposed to the specfile) + artifact_relpaths = [] # map of file extension to a list of files artifacts = {} # list of all files all_artifacts = [] - # list of all files with their maven repo path + # list of all files with their repo path all_artifacts_with_path = [] # makes generating relative paths easier self.pathinfo = koji.PathInfo(topdir='') if task: - # called as a subtask of a maven build + # called as a subtask of a build artifact_paths = self.session.listTaskOutput(task['id']) for artifact_path in artifact_paths: @@ -1343,7 +1340,8 @@ class WrapperRPMTask(BaseTaskHandler): # Exclude log files for consistency with the output of listArchives() used below continue relpath = os.path.join(self.pathinfo.task(task['id']), artifact_path)[1:] - artifacts.setdefault(ext, []).append(relpath) + artifact_relpaths.append(relpath) + artifacts.setdefault(ext, []).append(artifact_name) all_artifacts.append(artifact_name) all_artifacts_with_path.append(artifact_path) else: @@ -1352,61 +1350,67 @@ class WrapperRPMTask(BaseTaskHandler): if not build['state'] == koji.BUILD_STATES['COMPLETE']: raise koji.BuildError, 'cannot call wrapperRPM on a build that did not complete successfully' - maven_info = self.session.getMavenBuild(build['id'], strict=True) - # get the list of files from the build instead of the task, because the task output directory may # have already been cleaned up - build_artifacts = self.session.listArchives(buildID=build['id'], type='maven') + if maven_info: + build_artifacts = self.session.listArchives(buildID=build['id'], type='maven') + elif win_info: + build_artifacts = self.session.listArchives(buildID=build['id'], type='win') + else: + raise koji.BuildError, 'unsupported build type' for artifact in build_artifacts: artifact_name = artifact['filename'] base, ext = os.path.splitext(artifact_name) + artifacts.setdefault(ext, []).append(artifact_name) + all_artifacts.append(artifact_name) if ext == '.log': # listArchives() should never return .log files, but we check for completeness continue - relpath = os.path.join(self.pathinfo.mavenbuild(build, maven_info), artifact_name)[1:] - artifacts.setdefault(ext, []).append(relpath) - all_artifacts.append(artifact_name) - repopath = os.path.join(self.pathinfo.mavenrepo(maven_info, artifact), artifact_name)[len('/maven2/'):] - all_artifacts_with_path.append(repopath) + if maven_info: + relpath = os.path.join(self.pathinfo.mavenbuild(build, maven_info), artifact_name)[1:] + artifact_relpaths.append(relpath) + repopath = self.pathinfo.mavenfile(artifact) + all_artifacts_with_path.append(repopath) + elif win_info: + repopath = self.pathinfo.winfile(artifact) + relpath = os.path.join(self.pathinfo.winbuild(build), repopath)[1:] + artifact_relpaths.append(relpath) + all_artifacts_with_path.append(repopath) + else: + # can't happen + assert False if not artifacts: raise koji.BuildError, 'no output found for %s' % (task and koji.taskLabel(task) or koji.buildLabel(build)) - artifacts_base = {} - # construct a map of just basenames to pass to the template - for key, vals in artifacts.items(): - # sort the lists while we're at it - vals.sort() - artifacts_base[key] = [os.path.basename(val) for val in vals] - - values['artifacts'] = artifacts_base + values['artifacts'] = artifacts values['all_artifacts'] = all_artifacts values['all_artifacts_with_path'] = all_artifacts_with_path if build: self.copy_fields(build, values, 'epoch', 'name', 'version', 'release') - if not maven_info: - maven_info = self.session.getMavenBuild(build['id'], strict=True) - values['maven_info'] = maven_info + if maven_info: + values['maven_info'] = maven_info + elif win_info: + values['win_info'] = win_info + else: + # can't happen + assert False else: - # Get the pom info from the first pom and convert it to build format - # so we can use it as substitution variables in the template. - # If there are no poms, populate the values dict with empty placeholders. If the spec file - # doesn't require these variables, it should work. If it does, then the rpmbuild will - # likely fail. - poms = artifacts.get('.pom', []) - if poms: - pom_path = self.localPath(poms[0]) - pom_info = koji.parse_pom(pom_path) - maven_info = koji.pom_to_maven_info(pom_info) + task_result = self.session.getTaskResult(task['id']) + if task['method'] == 'buildMaven': + maven_info = task_result['maven_info'] maven_nvr = koji.maven_info_to_nvr(maven_info) maven_nvr['release'] = '0.scratch' self.copy_fields(maven_nvr, values, 'epoch', 'name', 'version', 'release') values['maven_info'] = maven_info + elif task['method'] == 'vmExec': + self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release') + values['win_info'] = {'platform': task_result['platform']} else: - values.update({'epoch': None, 'name': '', 'version': '', 'release': ''}) - values['maven_info'] = {'group_id': '', 'artifact_id': '', 'version': ''} + # can't happen + assert False scm = SCM(spec_url) scm.assert_allowed(self.options.allowed_scms) @@ -1442,11 +1446,19 @@ class WrapperRPMTask(BaseTaskHandler): # set to the rpm _sourcedir so other files in the SCM may be referenced in the # specfile as well. specdir = os.path.dirname(spec_template) - for key, filepaths in artifacts.items(): - for filepath in filepaths: - localpath = self.localPath(filepath) - shutil.copy(localpath, specdir) + for relpath in artifact_relpaths: + localpath = self.localPath(relpath) + # RPM requires all SOURCE files in the srpm to be in the same directory, so + # we flatten any directory structure of the output files here. + # If multiple files in the build have the same basename, the last one + # listed will be the one that ends up in the srpm, which is rarely the right + # thing to do. In this case the build should be putting the output into a + # zipfile or tarball so that rpmbuild can then expand it in %setup, which + # will preserve the directory structure. + shutil.copy(localpath, specdir) + # change directory to the specdir to the template can reference files there + os.chdir(specdir) contents = Cheetah.Template.Template(file=spec_template, searchList=[values]).respond() diff --git a/cli/koji b/cli/koji index d95e37d3..44cade71 100755 --- a/cli/koji +++ b/cli/koji @@ -4347,6 +4347,8 @@ def handle_win_build(options, session, args): parser.add_option("--mem", type="int", help=_("Amount of memory (in megabytes) to allocate to the build VM " + \ "(requires admin access)")) + parser.add_option("--specfile", metavar="URL", + help=_("SCM URL of a spec file fragment to use to generate wrapper RPMs")) parser.add_option("--scratch", action="store_true", help=_("Perform a scratch build")) parser.add_option("--repo-id", type="int", help=_("Use a specific repo")) @@ -4382,7 +4384,7 @@ def handle_win_build(options, session, args): vm_name = args[2] opts = {} for key in ('winspec', 'patches', 'cpus', 'mem', - 'scratch', 'repo_id', 'skip_tag'): + 'specfile', 'scratch', 'repo_id', 'skip_tag'): val = getattr(build_opts, key) if val is not None: opts[key] = val diff --git a/docs/schema-upgrade-1.4-1.5.sql b/docs/schema-upgrade-1.4-1.5.sql index b8e2ea40..ccc4432a 100644 --- a/docs/schema-upgrade-1.4-1.5.sql +++ b/docs/schema-upgrade-1.4-1.5.sql @@ -8,6 +8,7 @@ INSERT INTO permissions (name) VALUES ('win-admin'); INSERT INTO channels (name) VALUES ('vm'); +insert into archivetypes (name, description, extensions) values ('spec', 'RPM spec file', 'spec'); insert into archivetypes (name, description, extensions) values ('exe', 'Windows executable', 'exe'); insert into archivetypes (name, description, extensions) values ('dll', 'Windows dynamic link library', 'dll'); insert into archivetypes (name, description, extensions) values ('lib', 'Windows import library', 'lib'); diff --git a/docs/schema.sql b/docs/schema.sql index 77b32b61..7ff4c29c 100644 --- a/docs/schema.sql +++ b/docs/schema.sql @@ -692,6 +692,7 @@ insert into archivetypes (name, description, extensions) values ('zip', 'Zip arc insert into archivetypes (name, description, extensions) values ('pom', 'Maven Project Object Management file', 'pom'); insert into archivetypes (name, description, extensions) values ('tar', 'Tar file', 'tar tar.gz tar.bz2'); insert into archivetypes (name, description, extensions) values ('xml', 'XML file', 'xml'); +insert into archivetypes (name, description, extensions) values ('spec', 'RPM spec file', 'spec'); insert into archivetypes (name, description, extensions) values ('exe', 'Windows executable', 'exe'); insert into archivetypes (name, description, extensions) values ('dll', 'Windows dynamic link library', 'dll'); insert into archivetypes (name, description, extensions) values ('lib', 'Windows import library', 'lib'); diff --git a/hub/kojihub.py b/hub/kojihub.py index 4d9cc601..f950a811 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4351,11 +4351,12 @@ def new_maven_build(build, maven_info): VALUES (%(build_id)i, %(group_id)s, %(artifact_id)s, %(version)s)""" _dml(insert, maven_info) -def new_win_build(build_id, win_info): +def new_win_build(build_info, win_info): """ Add Windows metadata to an existing build. win_info must contain a 'platform' key. """ + build_id = build_info['id'] current = get_win_build(build_id, strict=False) if current: if current['platform'] != win_info['platform']: @@ -4981,6 +4982,7 @@ def reset_build(build): #nothing to do return minfo = get_maven_build(binfo) + winfo = get_win_build(binfo) koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=binfo['state'], new=koji.BUILD_STATES['CANCELED'], info=binfo) q = """SELECT id FROM rpminfo WHERE build_id=%(id)i""" ids = _fetchMulti(q, binfo) @@ -4996,12 +4998,16 @@ def reset_build(build): for (archive_id,) in ids: delete = """DELETE FROM maven_archives WHERE archive_id=%(archive_id)i""" _dml(delete, locals()) + delete = """DELETE FROM win_archives WHERE archive_id=%(archive_id)i""" + _dml(delete, locals()) delete = """DELETE FROM buildroot_archives WHERE archive_id=%(archive_id)i""" _dml(delete, locals()) delete = """DELETE FROM archiveinfo WHERE build_id=%(id)i""" _dml(delete, binfo) delete = """DELETE FROM maven_builds WHERE build_id = %(id)i""" _dml(delete, binfo) + delete = """DELETE FROM win_builds WHERE build_id = %(id)i""" + _dml(delete, binfo) binfo['state'] = koji.BUILD_STATES['CANCELED'] update = """UPDATE build SET state=%(state)i, task_id=NULL WHERE id=%(id)i""" _dml(update, binfo) @@ -5010,6 +5016,7 @@ def reset_build(build): builddir = koji.pathinfo.build(binfo) if os.path.exists(builddir): dirs_to_clear.append(builddir) + # Windows files exist under the builddir, and will be removed with the rpms if minfo: mavendir = koji.pathinfo.mavenbuild(binfo, minfo) if os.path.exists(mavendir): @@ -8827,7 +8834,7 @@ class HostExports(object): os.rename(fn,dest) os.symlink(dest,fn) - def moveWinBuildToScratch(self, task_id, results): + def moveWinBuildToScratch(self, task_id, results, rpm_results): "Move a completed scratch build into place (not imported)" if not context.opts.get('EnableWin'): raise koji.GenericError, 'Windows support not enabled' @@ -8837,13 +8844,22 @@ class HostExports(object): task.assertHost(host.id) scratchdir = koji.pathinfo.scratch() username = get_user(task.getOwner())['name'] - destdir = "%s/%s/task_%s" % (scratchdir, username, task_id) + destdir = os.path.join(scratchdir, username, 'task_%s' % task_id) for relpath in results['output'].keys() + results['logs']: filename = os.path.join(koji.pathinfo.task(results['task_id']), relpath) dest = os.path.join(destdir, relpath) koji.ensuredir(os.path.dirname(dest)) os.rename(filename, dest) os.symlink(dest, filename) + if rpm_results: + for relpath in [rpm_results['srpm']] + rpm_results['rpms'] + \ + rpm_results['logs']: + filename = os.path.join(koji.pathinfo.task(rpm_results['task_id']), + relpath) + dest = os.path.join(destdir, 'rpms', relpath) + koji.ensuredir(os.path.dirname(dest)) + os.rename(filename, dest) + os.symlink(dest, filename) def initBuild(self,data): """Create a stub build entry. @@ -8913,10 +8929,10 @@ class HostExports(object): data['state'] = koji.BUILD_STATES['BUILDING'] data['completion_time'] = None build_id = new_build(data) - build_info['id'] = build_id - new_maven_build(build_info, maven_info) + data['id'] = build_id + new_maven_build(data, maven_info) - return build_id, build_info + return data def completeMavenBuild(self, task_id, build_id, maven_results, rpm_results): """Complete the Maven build.""" @@ -9014,10 +9030,11 @@ class HostExports(object): data['state'] = koji.BUILD_STATES['BUILDING'] data['completion_time'] = None build_id = new_build(data) - new_win_build(build_id, win_info) - return build_id + data['id'] = build_id + new_win_build(data, win_info) + return data - def completeWinBuild(self, task_id, build_id, results): + def completeWinBuild(self, task_id, build_id, results, rpm_results): """Complete a Windows build""" if not context.opts.get('EnableWin'): raise koji.GenericError, 'Windows support not enabled' @@ -9049,6 +9066,9 @@ class HostExports(object): import_build_log(os.path.join(task_dir, relpath), build_info, subdir=subdir) + if rpm_results: + _import_wrapper(rpm_results['task_id'], build_info, rpm_results) + # update build state st_complete = koji.BUILD_STATES['COMPLETE'] update = UpdateProcessor('build', clauses=['id=%(build_id)i'], diff --git a/koji/__init__.py b/koji/__init__.py index dfabc18c..df9824a9 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1403,7 +1403,8 @@ class PathInfo(object): group_path = maveninfo['group_id'].replace('.', '/') artifact_id = maveninfo['artifact_id'] version = maveninfo['version'] - return "%(group_path)s/%(artifact_id)s/%(version)s" % locals() + filename = maveninfo['filename'] + return "%(group_path)s/%(artifact_id)s/%(version)s/%(filename)s" % locals() def mavenrepo(self, build, maveninfo): """Return the directory where the Maven artifact exists in the per-tag Maven repo @@ -1411,7 +1412,7 @@ class PathInfo(object): group_path = maveninfo['group_id'].replace('.', '/') artifact_id = maveninfo['artifact_id'] version = maveninfo['version'] - return self.topdir + "/maven2/" + self.mavenfile(maveninfo) + return self.topdir + "/maven2/" + os.path.dirname(self.mavenfile(maveninfo)) def rpm(self,rpminfo): """Return the path (relative to build_dir) where an rpm belongs""" diff --git a/koji/tasks.py b/koji/tasks.py index 7063721a..6ccdc584 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -439,3 +439,19 @@ class DependantTask(BaseTaskHandler): subtasks.append(task_id) if subtasks: self.wait(subtasks, all=True) + +class MultiPlatformTask(BaseTaskHandler): + def buildWrapperRPM(self, spec_url, build_task_id, build_tag, build, repo_id, **opts): + task = self.session.getTaskInfo(build_task_id) + arglist = [spec_url, build_tag, build, task, {'repo_id': repo_id}] + + rpm_task_id = self.session.host.subtask(method='wrapperRPM', + arglist=arglist, + label='rpm', + parent=self.id, + arch='noarch', + **opts) + results = self.wait(rpm_task_id)[rpm_task_id] + results['task_id'] = rpm_task_id + + return results diff --git a/vm/kojivmd b/vm/kojivmd index eb707dd5..11bb7eb1 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -23,7 +23,7 @@ import koji import koji.util from koji.daemon import SCM, TaskManager -from koji.tasks import ServerExit, BaseTaskHandler +from koji.tasks import ServerExit, BaseTaskHandler, MultiPlatformTask import sys import logging import os @@ -274,7 +274,7 @@ class DaemonXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer): return response -class WinBuildTask(BaseTaskHandler): +class WinBuildTask(MultiPlatformTask): """ Spawns a vmExec task to run a build, and imports the output. """ @@ -331,10 +331,10 @@ class WinBuildTask(BaseTaskHandler): results = self.wait(task_id)[task_id] results['task_id'] = task_id - if opts.get('scratch'): - self.session.host.moveWinBuildToScratch(self.id, results) - else: - build_info = koji.util.dslice(results, ['name', 'version', 'release']) + build_info = None + if not opts.get('scratch'): + build_info = koji.util.dslice(results, ['name', 'version', 'release', 'epoch']) + build_info['package_name'] = build_info['name'] pkg_cfg = self.session.getPackageConfig(dest_tag['id'], build_info['name'], event=event_id) if not opts.get('skip_tag'): # Make sure package is on the list for this tag @@ -345,25 +345,38 @@ class WinBuildTask(BaseTaskHandler): raise koji.BuildError, "package %s is blocked for tag %s" \ % (build_info['name'], dest_tag['name']) - # epoch is rpm-specific, so doesn't have a lot of relevance here - # but a value is required by the data model - build_info['epoch'] = None - build_id = self.session.host.initWinBuild(self.id, build_info, - koji.util.dslice(results, ['platform'])) - try: - self.session.host.completeWinBuild(self.id, build_id, results) - except (SystemExit, ServerExit, KeyboardInterrupt): - raise - except: + build_info = self.session.host.initWinBuild(self.id, build_info, + koji.util.dslice(results, ['platform'])) + build_id = build_info['id'] + + try: + rpm_results = None + spec_url = opts.get('specfile') + if spec_url: + rpm_results = self.buildWrapperRPM(spec_url, task_id, build_tag, build_info, repo_id, + channel='default') + + if opts.get('scratch'): + self.session.host.moveWinBuildToScratch(self.id, results, rpm_results) + else: + self.session.host.completeWinBuild(self.id, build_id, results, rpm_results) + except (SystemExit, ServerExit, KeyboardInterrupt): + # we do not trap these + raise + except: + if not opts.get('scratch'): + # scratch builds do not get imported self.session.host.failBuild(self.id, build_id) - raise - if not opts.get('skip_tag') and not opts.get('scratch'): - task_id = self.session.host.subtask(method='tagBuild', + # reraise the exception + raise + + if not opts.get('scratch') and not opts.get('skip_tag'): + tag_task_id = self.session.host.subtask(method='tagBuild', arglist=[dest_tag['id'], build_id], label='tag', channel='default', parent=self.id) - self.wait(task_id) + self.wait(tag_task_id) class VMExecTask(BaseTaskHandler): """