particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add support for wrapping win-build output in rpms

Browse source
This commit is contained in:
Mike Bonnet 2010-09-17 11:27:33 -04:00
parent b9f6cc7024
commit 64df809ab6
8 changed files with 155 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

126
builder/kojid
View file

@ -31,7 +31,7 @@ import glob
import logging
import logging.handlers
from koji.daemon import incremental_upload, log_output, TaskManager, SCM
from koji.tasks import ServerExit, BaseTaskHandler
from koji.tasks import ServerExit, BaseTaskHandler, MultiPlatformTask
from koji.util import parseStatus, isSuccess
import os
import pwd
@ -1014,7 +1014,7 @@ class BuildArchTask(BaseTaskHandler):
return ret
class MavenTask(BaseTaskHandler):
class MavenTask(MultiPlatformTask):
Methods = ['maven']
@ -1072,13 +1072,14 @@ class MavenTask(BaseTaskHandler):
raise koji.BuildError, "package %s is blocked for tag %s" \
% (build_info['name'], dest_tag['name'])
self.build_id, build_info = self.session.host.initMavenBuild(self.id, build_info, maven_info)
build_info = self.session.host.initMavenBuild(self.id, build_info, maven_info)
self.build_id = build_info['id']
try:
rpm_results = None
spec_url = self.opts.get('specfile')
if spec_url:
rpm_results = self.buildWrapperRPM(spec_url, build_tag, build_info, repo_id)
rpm_results = self.buildWrapperRPM(spec_url, self.build_task_id, build_tag, build_info, repo_id)
if not self.opts.get('scratch'):
self.session.host.completeMavenBuild(self.id, self.build_id, maven_results, rpm_results)
@ -1100,20 +1101,6 @@ class MavenTask(BaseTaskHandler):
arch='noarch')
self.wait(tag_task_id)
def buildWrapperRPM(self, spec_url, build_tag, build, repo_id):
task = self.session.getTaskInfo(self.build_task_id)
arglist = [spec_url, build_tag, build, task, {'repo_id': repo_id}]
rpm_task_id = self.session.host.subtask(method='wrapperRPM',
arglist=arglist,
label='rpm',
parent=self.id,
arch='noarch')
results = self.wait(rpm_task_id)[rpm_task_id]
results['task_id'] = rpm_task_id
return results
class BuildMavenTask(BaseTaskHandler):
Methods = ['buildMaven']
@ -1318,22 +1305,32 @@ class WrapperRPMTask(BaseTaskHandler):
if not opts:
opts = {}
if not (build or task):
raise koji.BuildError, 'build and/or task must be specified'
values = {}
maven_info = None
if build:
maven_info = self.session.getMavenBuild(build['id'], strict=False)
win_info = self.session.getWinBuild(build['id'], strict=False)
else:
maven_info = None
win_info = None
# list of artifact paths relative to kojiroot (not exposed to the specfile)
artifact_relpaths = []
# map of file extension to a list of files
artifacts = {}
# list of all files
all_artifacts = []
# list of all files with their maven repo path
# list of all files with their repo path
all_artifacts_with_path = []
# makes generating relative paths easier
self.pathinfo = koji.PathInfo(topdir='')
if task:
# called as a subtask of a maven build
# called as a subtask of a build
artifact_paths = self.session.listTaskOutput(task['id'])
for artifact_path in artifact_paths:
@ -1343,7 +1340,8 @@ class WrapperRPMTask(BaseTaskHandler):
# Exclude log files for consistency with the output of listArchives() used below
continue
relpath = os.path.join(self.pathinfo.task(task['id']), artifact_path)[1:]
artifacts.setdefault(ext, []).append(relpath)
artifact_relpaths.append(relpath)
artifacts.setdefault(ext, []).append(artifact_name)
all_artifacts.append(artifact_name)
all_artifacts_with_path.append(artifact_path)
else:
@ -1352,61 +1350,67 @@ class WrapperRPMTask(BaseTaskHandler):
if not build['state'] == koji.BUILD_STATES['COMPLETE']:
raise koji.BuildError, 'cannot call wrapperRPM on a build that did not complete successfully'
maven_info = self.session.getMavenBuild(build['id'], strict=True)
# get the list of files from the build instead of the task, because the task output directory may
# have already been cleaned up
build_artifacts = self.session.listArchives(buildID=build['id'], type='maven')
if maven_info:
build_artifacts = self.session.listArchives(buildID=build['id'], type='maven')
elif win_info:
build_artifacts = self.session.listArchives(buildID=build['id'], type='win')
else:
raise koji.BuildError, 'unsupported build type'
for artifact in build_artifacts:
artifact_name = artifact['filename']
base, ext = os.path.splitext(artifact_name)
artifacts.setdefault(ext, []).append(artifact_name)
all_artifacts.append(artifact_name)
if ext == '.log':
# listArchives() should never return .log files, but we check for completeness
continue
relpath = os.path.join(self.pathinfo.mavenbuild(build, maven_info), artifact_name)[1:]
artifacts.setdefault(ext, []).append(relpath)
all_artifacts.append(artifact_name)
repopath = os.path.join(self.pathinfo.mavenrepo(maven_info, artifact), artifact_name)[len('/maven2/'):]
all_artifacts_with_path.append(repopath)
if maven_info:
relpath = os.path.join(self.pathinfo.mavenbuild(build, maven_info), artifact_name)[1:]
artifact_relpaths.append(relpath)
repopath = self.pathinfo.mavenfile(artifact)
all_artifacts_with_path.append(repopath)
elif win_info:
repopath = self.pathinfo.winfile(artifact)
relpath = os.path.join(self.pathinfo.winbuild(build), repopath)[1:]
artifact_relpaths.append(relpath)
all_artifacts_with_path.append(repopath)
else:
# can't happen
assert False
if not artifacts:
raise koji.BuildError, 'no output found for %s' % (task and koji.taskLabel(task) or koji.buildLabel(build))
artifacts_base = {}
# construct a map of just basenames to pass to the template
for key, vals in artifacts.items():
# sort the lists while we're at it
vals.sort()
artifacts_base[key] = [os.path.basename(val) for val in vals]
values['artifacts'] = artifacts_base
values['artifacts'] = artifacts
values['all_artifacts'] = all_artifacts
values['all_artifacts_with_path'] = all_artifacts_with_path
if build:
self.copy_fields(build, values, 'epoch', 'name', 'version', 'release')
if not maven_info:
maven_info = self.session.getMavenBuild(build['id'], strict=True)
values['maven_info'] = maven_info
if maven_info:
values['maven_info'] = maven_info
elif win_info:
values['win_info'] = win_info
else:
# can't happen
assert False
else:
# Get the pom info from the first pom and convert it to build format
# so we can use it as substitution variables in the template.
# If there are no poms, populate the values dict with empty placeholders. If the spec file
# doesn't require these variables, it should work. If it does, then the rpmbuild will
# likely fail.
poms = artifacts.get('.pom', [])
if poms:
pom_path = self.localPath(poms[0])
pom_info = koji.parse_pom(pom_path)
maven_info = koji.pom_to_maven_info(pom_info)
task_result = self.session.getTaskResult(task['id'])
if task['method'] == 'buildMaven':
maven_info = task_result['maven_info']
maven_nvr = koji.maven_info_to_nvr(maven_info)
maven_nvr['release'] = '0.scratch'
self.copy_fields(maven_nvr, values, 'epoch', 'name', 'version', 'release')
values['maven_info'] = maven_info
elif task['method'] == 'vmExec':
self.copy_fields(task_result, values, 'epoch', 'name', 'version', 'release')
values['win_info'] = {'platform': task_result['platform']}
else:
values.update({'epoch': None, 'name': '', 'version': '', 'release': ''})
values['maven_info'] = {'group_id': '', 'artifact_id': '', 'version': ''}
# can't happen
assert False
scm = SCM(spec_url)
scm.assert_allowed(self.options.allowed_scms)
@ -1442,11 +1446,19 @@ class WrapperRPMTask(BaseTaskHandler):
# set to the rpm _sourcedir so other files in the SCM may be referenced in the
# specfile as well.
specdir = os.path.dirname(spec_template)
for key, filepaths in artifacts.items():
for filepath in filepaths:
localpath = self.localPath(filepath)
shutil.copy(localpath, specdir)
for relpath in artifact_relpaths:
localpath = self.localPath(relpath)
# RPM requires all SOURCE files in the srpm to be in the same directory, so
# we flatten any directory structure of the output files here.
# If multiple files in the build have the same basename, the last one
# listed will be the one that ends up in the srpm, which is rarely the right
# thing to do. In this case the build should be putting the output into a
# zipfile or tarball so that rpmbuild can then expand it in %setup, which
# will preserve the directory structure.
shutil.copy(localpath, specdir)
# change directory to the specdir to the template can reference files there
os.chdir(specdir)
contents = Cheetah.Template.Template(file=spec_template,
searchList=[values]).respond()

4
cli/koji
View file

@ -4347,6 +4347,8 @@ def handle_win_build(options, session, args):
parser.add_option("--mem", type="int",
help=_("Amount of memory (in megabytes) to allocate to the build VM " + \
"(requires admin access)"))
parser.add_option("--specfile", metavar="URL",
help=_("SCM URL of a spec file fragment to use to generate wrapper RPMs"))
parser.add_option("--scratch", action="store_true",
help=_("Perform a scratch build"))
parser.add_option("--repo-id", type="int", help=_("Use a specific repo"))
@ -4382,7 +4384,7 @@ def handle_win_build(options, session, args):
vm_name = args[2]
opts = {}
for key in ('winspec', 'patches', 'cpus', 'mem',
'scratch', 'repo_id', 'skip_tag'):
'specfile', 'scratch', 'repo_id', 'skip_tag'):
val = getattr(build_opts, key)
if val is not None:
opts[key] = val

1
docs/schema-upgrade-1.4-1.5.sql
View file

@ -8,6 +8,7 @@ INSERT INTO permissions (name) VALUES ('win-admin');
INSERT INTO channels (name) VALUES ('vm');
insert into archivetypes (name, description, extensions) values ('spec', 'RPM spec file', 'spec');
insert into archivetypes (name, description, extensions) values ('exe', 'Windows executable', 'exe');
insert into archivetypes (name, description, extensions) values ('dll', 'Windows dynamic link library', 'dll');
insert into archivetypes (name, description, extensions) values ('lib', 'Windows import library', 'lib');

1
docs/schema.sql
View file

@ -692,6 +692,7 @@ insert into archivetypes (name, description, extensions) values ('zip', 'Zip arc
insert into archivetypes (name, description, extensions) values ('pom', 'Maven Project Object Management file', 'pom');
insert into archivetypes (name, description, extensions) values ('tar', 'Tar file', 'tar tar.gz tar.bz2');
insert into archivetypes (name, description, extensions) values ('xml', 'XML file', 'xml');
insert into archivetypes (name, description, extensions) values ('spec', 'RPM spec file', 'spec');
insert into archivetypes (name, description, extensions) values ('exe', 'Windows executable', 'exe');
insert into archivetypes (name, description, extensions) values ('dll', 'Windows dynamic link library', 'dll');
insert into archivetypes (name, description, extensions) values ('lib', 'Windows import library', 'lib');

38
hub/kojihub.py
View file

@ -4351,11 +4351,12 @@ def new_maven_build(build, maven_info):
VALUES (%(build_id)i, %(group_id)s, %(artifact_id)s, %(version)s)"""
_dml(insert, maven_info)
def new_win_build(build_id, win_info):
def new_win_build(build_info, win_info):
"""
Add Windows metadata to an existing build.
win_info must contain a 'platform' key.
"""
build_id = build_info['id']
current = get_win_build(build_id, strict=False)
if current:
if current['platform'] != win_info['platform']:
@ -4981,6 +4982,7 @@ def reset_build(build):
#nothing to do
return
minfo = get_maven_build(binfo)
winfo = get_win_build(binfo)
koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=binfo['state'], new=koji.BUILD_STATES['CANCELED'], info=binfo)
q = """SELECT id FROM rpminfo WHERE build_id=%(id)i"""
ids = _fetchMulti(q, binfo)
@ -4996,12 +4998,16 @@ def reset_build(build):
for (archive_id,) in ids:
delete = """DELETE FROM maven_archives WHERE archive_id=%(archive_id)i"""
_dml(delete, locals())
delete = """DELETE FROM win_archives WHERE archive_id=%(archive_id)i"""
_dml(delete, locals())
delete = """DELETE FROM buildroot_archives WHERE archive_id=%(archive_id)i"""
_dml(delete, locals())
delete = """DELETE FROM archiveinfo WHERE build_id=%(id)i"""
_dml(delete, binfo)
delete = """DELETE FROM maven_builds WHERE build_id = %(id)i"""
_dml(delete, binfo)
delete = """DELETE FROM win_builds WHERE build_id = %(id)i"""
_dml(delete, binfo)
binfo['state'] = koji.BUILD_STATES['CANCELED']
update = """UPDATE build SET state=%(state)i, task_id=NULL WHERE id=%(id)i"""
_dml(update, binfo)
@ -5010,6 +5016,7 @@ def reset_build(build):
builddir = koji.pathinfo.build(binfo)
if os.path.exists(builddir):
dirs_to_clear.append(builddir)
# Windows files exist under the builddir, and will be removed with the rpms
if minfo:
mavendir = koji.pathinfo.mavenbuild(binfo, minfo)
if os.path.exists(mavendir):
@ -8827,7 +8834,7 @@ class HostExports(object):
os.rename(fn,dest)
os.symlink(dest,fn)
def moveWinBuildToScratch(self, task_id, results):
def moveWinBuildToScratch(self, task_id, results, rpm_results):
"Move a completed scratch build into place (not imported)"
if not context.opts.get('EnableWin'):
raise koji.GenericError, 'Windows support not enabled'
@ -8837,13 +8844,22 @@ class HostExports(object):
task.assertHost(host.id)
scratchdir = koji.pathinfo.scratch()
username = get_user(task.getOwner())['name']
destdir = "%s/%s/task_%s" % (scratchdir, username, task_id)
destdir = os.path.join(scratchdir, username, 'task_%s' % task_id)
for relpath in results['output'].keys() + results['logs']:
filename = os.path.join(koji.pathinfo.task(results['task_id']), relpath)
dest = os.path.join(destdir, relpath)
koji.ensuredir(os.path.dirname(dest))
os.rename(filename, dest)
os.symlink(dest, filename)
if rpm_results:
for relpath in [rpm_results['srpm']] + rpm_results['rpms'] + \
rpm_results['logs']:
filename = os.path.join(koji.pathinfo.task(rpm_results['task_id']),
relpath)
dest = os.path.join(destdir, 'rpms', relpath)
koji.ensuredir(os.path.dirname(dest))
os.rename(filename, dest)
os.symlink(dest, filename)
def initBuild(self,data):
"""Create a stub build entry.
@ -8913,10 +8929,10 @@ class HostExports(object):
data['state'] = koji.BUILD_STATES['BUILDING']
data['completion_time'] = None
build_id = new_build(data)
build_info['id'] = build_id
new_maven_build(build_info, maven_info)
data['id'] = build_id
new_maven_build(data, maven_info)
return build_id, build_info
return data
def completeMavenBuild(self, task_id, build_id, maven_results, rpm_results):
"""Complete the Maven build."""
@ -9014,10 +9030,11 @@ class HostExports(object):
data['state'] = koji.BUILD_STATES['BUILDING']
data['completion_time'] = None
build_id = new_build(data)
new_win_build(build_id, win_info)
return build_id
data['id'] = build_id
new_win_build(data, win_info)
return data
def completeWinBuild(self, task_id, build_id, results):
def completeWinBuild(self, task_id, build_id, results, rpm_results):
"""Complete a Windows build"""
if not context.opts.get('EnableWin'):
raise koji.GenericError, 'Windows support not enabled'
@ -9049,6 +9066,9 @@ class HostExports(object):
import_build_log(os.path.join(task_dir, relpath),
build_info, subdir=subdir)
if rpm_results:
_import_wrapper(rpm_results['task_id'], build_info, rpm_results)
# update build state
st_complete = koji.BUILD_STATES['COMPLETE']
update = UpdateProcessor('build', clauses=['id=%(build_id)i'],

5
koji/__init__.py
View file

@ -1403,7 +1403,8 @@ class PathInfo(object):
group_path = maveninfo['group_id'].replace('.', '/')
artifact_id = maveninfo['artifact_id']
version = maveninfo['version']
return "%(group_path)s/%(artifact_id)s/%(version)s" % locals()
filename = maveninfo['filename']
return "%(group_path)s/%(artifact_id)s/%(version)s/%(filename)s" % locals()
def mavenrepo(self, build, maveninfo):
"""Return the directory where the Maven artifact exists in the per-tag Maven repo
@ -1411,7 +1412,7 @@ class PathInfo(object):
group_path = maveninfo['group_id'].replace('.', '/')
artifact_id = maveninfo['artifact_id']
version = maveninfo['version']
return self.topdir + "/maven2/" + self.mavenfile(maveninfo)
return self.topdir + "/maven2/" + os.path.dirname(self.mavenfile(maveninfo))
def rpm(self,rpminfo):
"""Return the path (relative to build_dir) where an rpm belongs"""

16
koji/tasks.py
View file

@ -439,3 +439,19 @@ class DependantTask(BaseTaskHandler):
subtasks.append(task_id)
if subtasks:
self.wait(subtasks, all=True)
class MultiPlatformTask(BaseTaskHandler):
def buildWrapperRPM(self, spec_url, build_task_id, build_tag, build, repo_id, **opts):
task = self.session.getTaskInfo(build_task_id)
arglist = [spec_url, build_tag, build, task, {'repo_id': repo_id}]
rpm_task_id = self.session.host.subtask(method='wrapperRPM',
arglist=arglist,
label='rpm',
parent=self.id,
arch='noarch',
**opts)
results = self.wait(rpm_task_id)[rpm_task_id]
results['task_id'] = rpm_task_id
return results

53
vm/kojivmd
View file

@ -23,7 +23,7 @@
import koji
import koji.util
from koji.daemon import SCM, TaskManager
from koji.tasks import ServerExit, BaseTaskHandler
from koji.tasks import ServerExit, BaseTaskHandler, MultiPlatformTask
import sys
import logging
import os
@ -274,7 +274,7 @@ class DaemonXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer):
return response
class WinBuildTask(BaseTaskHandler):
class WinBuildTask(MultiPlatformTask):
"""
Spawns a vmExec task to run a build, and imports the output.
"""
@ -331,10 +331,10 @@ class WinBuildTask(BaseTaskHandler):
results = self.wait(task_id)[task_id]
results['task_id'] = task_id
if opts.get('scratch'):
self.session.host.moveWinBuildToScratch(self.id, results)
else:
build_info = koji.util.dslice(results, ['name', 'version', 'release'])
build_info = None
if not opts.get('scratch'):
build_info = koji.util.dslice(results, ['name', 'version', 'release', 'epoch'])
build_info['package_name'] = build_info['name']
pkg_cfg = self.session.getPackageConfig(dest_tag['id'], build_info['name'], event=event_id)
if not opts.get('skip_tag'):
# Make sure package is on the list for this tag
@ -345,25 +345,38 @@ class WinBuildTask(BaseTaskHandler):
raise koji.BuildError, "package %s is blocked for tag %s" \
% (build_info['name'], dest_tag['name'])
# epoch is rpm-specific, so doesn't have a lot of relevance here
# but a value is required by the data model
build_info['epoch'] = None
build_id = self.session.host.initWinBuild(self.id, build_info,
koji.util.dslice(results, ['platform']))
try:
self.session.host.completeWinBuild(self.id, build_id, results)
except (SystemExit, ServerExit, KeyboardInterrupt):
raise
except:
build_info = self.session.host.initWinBuild(self.id, build_info,
koji.util.dslice(results, ['platform']))
build_id = build_info['id']
try:
rpm_results = None
spec_url = opts.get('specfile')
if spec_url:
rpm_results = self.buildWrapperRPM(spec_url, task_id, build_tag, build_info, repo_id,
channel='default')
if opts.get('scratch'):
self.session.host.moveWinBuildToScratch(self.id, results, rpm_results)
else:
self.session.host.completeWinBuild(self.id, build_id, results, rpm_results)
except (SystemExit, ServerExit, KeyboardInterrupt):
# we do not trap these
raise
except:
if not opts.get('scratch'):
# scratch builds do not get imported
self.session.host.failBuild(self.id, build_id)
raise
if not opts.get('skip_tag') and not opts.get('scratch'):
task_id = self.session.host.subtask(method='tagBuild',
# reraise the exception
raise
if not opts.get('scratch') and not opts.get('skip_tag'):
tag_task_id = self.session.host.subtask(method='tagBuild',
arglist=[dest_tag['id'], build_id],
label='tag',
channel='default',
parent=self.id)
self.wait(task_id)
self.wait(tag_task_id)
class VMExecTask(BaseTaskHandler):
"""