support specifying parameters of a buildrequires, including type, which allows builds of one type to depend on files from builds of a different type

This commit is contained in:
Mike Bonnet 2010-08-06 08:34:24 -04:00
parent 4e8eb0c4e3
commit f619543ac8
9 changed files with 234 additions and 74 deletions

View file

@ -824,17 +824,6 @@ class BuildTask(BaseTaskHandler):
raise koji.BuildError, "No matching arches were found"
return archdict.keys()
def getRepo(self, tag):
"""Get repo to use for builds"""
repo_info = self.session.getRepo(tag)
if not repo_info:
#wait for it
task_id = self.session.host.subtask(method='waitrepo',
arglist=[tag, None, None],
parent=self.id)
repo_info = self.wait(task_id)[task_id]
return repo_info
def runBuilds(self, srpm, build_tag, archlist, repo_id):
self.logger.debug("Spawning jobs for arches: %r" % (archlist))
subtasks = {}

View file

@ -3432,11 +3432,32 @@ def get_archive(archive_id, strict=False):
filename: name of the archive (string)
size: size of the archive (integer)
md5sum: md5sum of the archive (string)
If the archive is part of a Maven build, the following keys will be included:
group_id
artifact_id
version
If the archive is part of a Windows builds, the following keys will be included:
relpath
platforms
flags
"""
fields = ('id', 'type_id', 'build_id', 'buildroot_id', 'filename', 'size', 'md5sum')
select = """SELECT %s FROM archiveinfo
WHERE id = %%(archive_id)i""" % ', '.join(fields)
return _singleRow(select, locals(), fields, strict=strict)
archive = _singleRow(select, locals(), fields, strict=strict)
if not archive:
# strict is taken care of by _singleRow()
return None
maven_info = get_maven_archive(archive_id)
if maven_info:
del maven_info['archive_id']
archive.update(maven_info)
win_info = get_win_archive(archive_id)
if win_info:
del win_info['archive_id']
archive.update(win_info)
return archive
def get_maven_archive(archive_id, strict=False):
"""

View file

@ -1394,17 +1394,24 @@ class PathInfo(object):
"""Return the relative path from the winbuild directory where the
file identified by wininfo is located."""
filepath = wininfo['filename']
if wininfo.get('relpath'):
if wininfo['relpath']:
filepath = wininfo['relpath'] + '/' + filepath
return filepath
def mavenfile(self, maveninfo):
"""Return the relative path the file exists in the per-tag Maven repo"""
group_path = maveninfo['group_id'].replace('.', '/')
artifact_id = maveninfo['artifact_id']
version = maveninfo['version']
return "%(group_path)s/%(artifact_id)s/%(version)s" % locals()
def mavenrepo(self, build, maveninfo):
"""Return the directory where the Maven artifact exists in the per-tag Maven repo
(/mnt/koji/repos/tag-name/repo-id/maven2/)"""
group_path = maveninfo['group_id'].replace('.', '/')
artifact_id = maveninfo['artifact_id']
version = maveninfo['version']
return self.topdir + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s" % locals())
return self.topdir + "/maven2/" + self.mavenfile(maveninfo)
def rpm(self,rpminfo):
"""Return the path (relative to build_dir) where an rpm belongs"""

View file

@ -741,8 +741,7 @@ class TaskManager(object):
self.logger.debug("available capacities for bin: %r" % bin_avail)
median = bin_avail[(len(bin_avail)-1)/2]
self.logger.debug("ours: %.2f, median: %.2f" % (our_avail, median))
if our_avail < median:
self.logger.debug("Skipping - available capacity in lower half")
if not self.checkRelAvail(bin_avail, our_avail):
#decline for now and give the upper half a chance
return False
#otherwise, we attempt to open the task
@ -753,6 +752,19 @@ class TaskManager(object):
raise Exception, "Invalid task state reported by server"
return False
def checkRelAvail(self, bin_avail, avail):
"""
Check our available capacity against the capacity of other hosts in this bin.
Return True if we should take a task, False otherwise.
"""
median = bin_avail[(len(bin_avail)-1)/2]
self.logger.debug("ours: %.2f, median: %.2f" % (avail, median))
if avail >= median:
return True
else:
self.logger.debug("Skipping - available capacity in lower half")
return False
def _waitTask(self, task_id, pid=None):
"""Wait (nohang) on the task, return true if finished"""
if pid is None:

View file

@ -320,6 +320,21 @@ class BaseTaskHandler(object):
raise koji.BuildError, "host %s (%s) does not support any arches of tag %s (%s)" % \
(host['name'], ', '.join(host_arches), tag['name'], ', '.join(tag_arches))
def getRepo(self, tag):
"""
Get the active repo for the given tag. If there is no repo available,
wait for a repo to be created.
"""
repo_info = self.session.getRepo(tag)
if not repo_info:
#wait for it
task_id = self.session.host.subtask(method='waitrepo',
arglist=[tag, None, None],
parent=self.id)
repo_info = self.wait(task_id)[task_id]
return repo_info
class FakeTask(BaseTaskHandler):
Methods = ['someMethod']
Foreground = True

View file

@ -304,7 +304,7 @@ class WindowsBuild(object):
self.release = None
self.description = None
self.platform = None
self.buildrequires = {}
self.buildrequires = []
self.provides = []
self.shell = None
self.execute = []
@ -360,8 +360,22 @@ class WindowsBuild(object):
self.platform = conf.get('building', 'platform')
# buildrequires and provides are multi-valued (space-separated)
for br in conf.get('building', 'buildrequires').split():
# buildrequires is a space-separated list
# each item in the list is in the format:
# pkgname[:opt1:opt2=val2:...]
# the options are put into a dict
# if the option has no =val, the value in the dict will be None
if br:
self.buildrequires[br] = {}
br = br.split(':')
bropts = {}
for opt in br[1:]:
if '=' in opt:
key, val = opt.split('=', 1)
else:
key = opt
val = None
bropts[key] = val
self.buildrequires.append((br[0], bropts))
for prov in conf.get('building', 'provides').split():
if prov:
self.provides.append(prov)
@ -389,12 +403,9 @@ class WindowsBuild(object):
self.output[filename] = metadata
self.logs.extend([e.strip() for e in conf.get('files', 'logs').split('\n') if e])
def fetchFile(self, basedir, buildinfo, fileinfo, type=None):
def fetchFile(self, basedir, buildinfo, fileinfo, type):
"""Download the file from buildreq, at filepath, into the basedir"""
if type == 'win':
destpath = os.path.join(basedir, fileinfo['relpath'], fileinfo['filename'])
else:
raise BuildError, 'unsupported file type: %s' % type
destpath = os.path.join(basedir, fileinfo['localpath'])
ensuredir(os.path.dirname(destpath))
destfile = file(destpath, 'w')
offset = 0
@ -410,28 +421,28 @@ class WindowsBuild(object):
checksum.update(data)
destfile.close()
digest = checksum.hexdigest()
if digest != fileinfo['md5sum']:
# rpms don't have a md5sum in the fileinfo, but check it for everything else
if ('md5sum' in fileinfo) and (digest != fileinfo['md5sum']):
raise BuildError, 'md5 checksum validation failed for %s, %s (computed) != %s (provided)' % \
(destpath, digest, fileinfo['md5sum'])
self.logger.info('Retrieved %s (%s bytes, md5: %s)', destpath, offset, digest)
def fetchBuildReqs(self):
"""Retrieve buildrequires listed in the spec file"""
for buildreq, brinfo in self.buildrequires.items():
for buildreq, brinfo in self.buildrequires:
# if no type is specified in the options, default to win
brtype = brinfo.get('type', 'win')
buildinfo = self.server.getLatestBuild(self.build_tag, buildreq,
self.task_opts.get('event_id'))
br_dir = os.path.join(self.buildreq_dir, buildreq)
self.task_opts.get('repo_id'))
br_dir = os.path.join(self.buildreq_dir, buildreq, brtype)
ensuredir(br_dir)
brinfo['dir'] = br_dir
brfiles = []
brinfo['files'] = brfiles
buildfiles = self.server.getFileList(buildinfo['id'], 'win')
buildfiles = self.server.getFileList(buildinfo['id'], brtype, brinfo)
for fileinfo in buildfiles:
self.fetchFile(br_dir, buildinfo, fileinfo, 'win')
if fileinfo['relpath']:
brfiles.append(os.path.join(fileinfo['relpath'], fileinfo['filename']))
else:
brfiles.append(fileinfo['filename'])
self.fetchFile(br_dir, buildinfo, fileinfo, brtype)
brfiles.append(fileinfo['localpath'])
def build(self):
if self.shell in ('cmd', 'cmd.exe'):
@ -455,14 +466,22 @@ class WindowsBuild(object):
"""Do the build: run the execute line(s) with cmd.exe"""
tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp', suffix='.bat', dir='/cygdrive/c/Windows/Temp')
script = os.fdopen(tmpfd, 'w')
for buildreq, brinfo in self.buildrequires.items():
for buildreq, brinfo in self.buildrequires:
buildreq = self.varname(buildreq)
ret, output = run(['cygpath', '-wa', brinfo['dir']], log=False, fatal=True)
br_dir = output.strip()
script.write('set %s_dir=%s\r\n' % (buildreq, br_dir))
files = ' '.join(brinfo['files'])
files.replace('/', '\\')
script.write('set %s_files=%s\r\n' % (buildreq, files))
if brinfo.get('type'):
# if the spec file qualifies the buildreq with a type,
# the env. var is named buildreq_type_{dir,files}
script.write('set %s_%s_dir=%s\r\n' % (buildreq, brinfo['type'], br_dir))
script.write('set %s_%s_files=%s\r\n' % (buildreq, brinfo['type'], files))
else:
# otherwise it's just buildreq_{dir,files}
script.write('set %s_dir=%s\r\n' % (buildreq, br_dir))
script.write('set %s_files=%s\r\n' % (buildreq, files))
script.write('\r\n')
for cmd in self.execute:
script.write(cmd)
script.write('\r\n')
@ -476,10 +495,14 @@ class WindowsBuild(object):
"""Do the build: run the execute line(s) with bash"""
tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp.', dir='/tmp')
script = os.fdopen(tmpfd, 'w')
for buildreq, brinfo in self.buildrequires.items():
for buildreq, brinfo in self.buildrequires:
buildreq = self.varname(buildreq)
script.write("export %s_dir='%s'\n" % (buildreq, brinfo['dir']))
script.write("export %s_files='" % buildreq)
if brinfo.get('type'):
script.write("export %s_%s_dir='%s'\n" % (buildreq, brinfo['type'], brinfo['dir']))
script.write("export %s_%s_files='" % (buildreq, brinfo['type']))
else:
script.write("export %s_dir='%s'\n" % (buildreq, brinfo['dir']))
script.write("export %s_files='" % buildreq)
for filename in brinfo['files']:
script.write(filename)
script.write('\n')
@ -664,19 +687,26 @@ def incremental_upload(server, handler):
else:
time.sleep(1)
def flunk(server, logfile):
def flunk(server, handler):
"""do the right thing when a build fails"""
global logfd
logging.getLogger('koji.vm').error('error running build', exc_info=True)
tb = ''.join(traceback.format_exception(*sys.exc_info()))
handler.active = False
if server is not None:
try:
logfd.flush()
upload_file(server, os.path.dirname(logfile),
os.path.basename(logfile))
upload_file(server, os.path.dirname(handler.baseFilename),
os.path.basename(handler.baseFilename))
except:
pass
server.failTask(tb)
log_local('error calling upload_file()')
while True:
try:
# this is the very last thing we do, keep trying as long as we can
server.failTask(tb)
break
except:
log_local('error calling server.failTask()')
sys.exit(1)
logfd = None
@ -715,6 +745,7 @@ if __name__ == '__main__':
thread = threading.Thread(target=incremental_upload,
args=(server, handler))
thread.daemon = True
thread.start()
build = WindowsBuild(server)
@ -732,5 +763,5 @@ if __name__ == '__main__':
server.closeTask(results)
except:
flunk(server, logfile)
flunk(server, handler)
sys.exit(0)

View file

@ -41,6 +41,7 @@ import threading
import base64
import pwd
import urlgrabber
import fnmatch
from ConfigParser import ConfigParser
from optparse import OptionParser
try:
@ -301,6 +302,7 @@ class WinBuildTask(BaseTaskHandler):
repo_id = opts.get('repo_id')
if repo_id:
repo_info = session.repoInfo(repo_id)
event_id = repo_info['create_event']
if not repo_info:
raise koji.BuildError, 'invalid repo ID: %s' % repo_id
policy_data = {
@ -314,11 +316,12 @@ class WinBuildTask(BaseTaskHandler):
if not opts.get('skip_tag'):
policy_data['tag'] = dest_tag['id']
self.session.host.assertPolicy('build_from_repo_id', policy_data)
event_id = repo_info['create_event']
else:
event_id = self.session.getLastEvent()['id']
repo_info = self.getRepo(build_tag['id'])
repo_id = repo_info['id']
event_id = None
subopts['event_id'] = event_id
subopts['repo_id'] = repo_id
task_opts = koji.util.dslice(opts, ['timeout', 'cpus', 'mem'], strict=False)
task_id = self.session.host.subtask(method='vmExec',
@ -463,39 +466,110 @@ class VMExecTask(BaseTaskHandler):
"""
return self.task_info
def getLatestBuild(self, tag, package, event=None):
def getLatestBuild(self, tag, package, repo_id):
"""
Get information about the latest build of package "package" in tag "tag".
"""
builds = self.session.getLatestBuilds(tag, package=package, event=event)
repo_info = self.session.repoInfo(repo_id, strict=True)
builds = self.session.getLatestBuilds(tag, package=package,
event=repo_info['create_event'])
if not builds:
raise koji.BuildError, 'no build of package %s in tag %s' % (package, tag)
return builds[0]
build = builds[0]
maven_build = self.session.getMavenBuild(build['id'])
if maven_build:
del maven_build['build_id']
build.update(maven_build)
win_build = self.session.getWinBuild(build['id'])
if win_build:
del win_build['build_id']
build.update(win_build)
return build
def getFileList(self, buildID, type=None):
def getFileList(self, buildID, type, typeopts):
"""
Get the file list for the latest build of the package "package" in tag "tag".
If type is specified, include the extended information for archives of that type.
Get the list of files of "type" for the latest build of the package "package" in tag "tag".
typeopts is a dict that is used to filter the file list.
typeopts is checked for:
patterns: comma-separated list of path/filename patterns (as used by fnmatch)
to filter the results with
If type is 'rpm', typeopts is checked for:
arches: comma-separated list of arches to include in output
If type is 'maven', typeopts is checked for:
group_ids: Maven group IDs to include in the output
artifact_ids: Maven artifact IDs to include in the output
versions: Maven versions to include in the output
If type is 'win', typeopts is checked for:
platforms: comma-separated list of platforms
flags: comma-separated list of flags
"""
return self.session.listArchives(buildID=buildID, type=type)
if not typeopts:
typeopts = {}
if type == 'rpm':
arches = None
if typeopts.get('arches'):
arches = typeopts['arches'].split(',')
files = self.session.listRPMs(buildID=buildID, arches=arches)
else:
files = self.session.listArchives(buildID=buildID, type=type)
for fileinfo in files:
if type == 'rpm':
filepath = koji.pathinfo.rpm(fileinfo)
elif type == 'maven':
filepath = koji.pathinfo.mavenfile(fileinfo)
elif type == 'win':
filepath = koji.pathinfo.winfile(fileinfo)
else:
# XXX support other file types when available
filepath = fileinfo['filename']
fileinfo['localpath'] = filepath
if typeopts.get('patterns'):
to_filter = files
files = []
patterns = typeopts['patterns'].split(',')
for fileinfo in to_filter:
for pattern in patterns:
if fnmatch.fnmatch(fileinfo['localpath'], pattern):
files.append(fileinfo)
break
if type == 'maven':
if typeopts.get('group_ids'):
group_ids = typeopts['group_ids'].split(',')
files = [f for f in files if f['group_id'] in group_ids]
if typeopts.get('artifact_ids'):
artifact_ids = typeopts['artifact_ids'].split(',')
files = [f for f in files if f['artifact_id'] in artifact_ids]
if typeopts.get('versions'):
versions = typeopts['versions'].split(',')
files = [f for f in files if f['version'] in versions]
if type == 'win':
if typeopts.get('platforms'):
platforms = typeopts['platforms'].split(',')
files = [f for f in files if set(f['platforms'].split()).intersection(platforms)]
if typeopts.get('flags'):
flags = typeopts['flags'].split(',')
files = [f for f in files if set(f['flags'].split()).intersection(flags)]
return files
def localCache(self, buildinfo, fileinfo, type=None):
def localCache(self, buildinfo, fileinfo, type):
"""
Access a file in the local cache. If the file does not exist, it's downloaded
from the server. Returns an open file object.
"""
local_pi = koji.PathInfo(self.buildreq_dir)
if type == 'win':
localpath = os.path.join(local_pi.winbuild(buildinfo),
fileinfo['relpath'], fileinfo['filename'])
else:
raise koji.BuildError, 'unsupported file type: %s' % type
# fileinfo['localpath'] is set by getFileList()
localpath = os.path.join(self.buildreq_dir, type, fileinfo['localpath'])
if not os.path.isfile(localpath):
remote_pi = koji.PathInfo(self.options.topurl)
if type == 'win':
if type == 'rpm':
remote_url = remote_pi.build(buildinfo) + '/' + \
remote_pi.rpm(fileinfo)
elif type == 'maven':
# the relevant Maven info is included in the buildinfo
remote_url = remote_pi.mavenbuild(buildinfo, buildinfo) + '/' + \
fileinfo['filename']
elif type == 'win':
remote_url = remote_pi.winbuild(buildinfo) + '/' + \
fileinfo['relpath'] + '/' + fileinfo['filename']
remote_pi.winfile(fileinfo)
else:
raise koji.BuildError, 'unsupported file type: %s' % type
koji.ensuredir(os.path.dirname(localpath))
@ -503,14 +577,14 @@ class VMExecTask(BaseTaskHandler):
return file(localpath, 'r')
def getFile(self, buildinfo, archiveinfo, offset, length, type=None):
def getFile(self, buildinfo, archiveinfo, offset, length, type):
"""
Get the contents of the file indicated by fileinfo, returning a maximum of
"length" bytes starting at "offset". Contents are returned base64-encoded.
"""
offset = int(offset)
length = int(length)
fileobj = self.localCache(buildinfo, archiveinfo, type=type)
fileobj = self.localCache(buildinfo, archiveinfo, type)
try:
fileobj.seek(offset)
data = fileobj.read(length)
@ -782,6 +856,13 @@ class VMTaskManager(TaskManager):
"""See if we have enough space to accept another job"""
return self.checkDisk() and self.checkMem()
def checkRelAvail(self, bin_avail, avail):
"""
Always return True, since we may be the only daemon with access
to the VM required to process this task.
"""
return True
def takeTask(self, task):
"""
Verify that this builder can handle the task before claiming it.

View file

@ -26,13 +26,13 @@
</tr>
#if $maveninfo
<tr>
<th>Maven groupId</th><td>$maveninfo.group_id</td>
<th>Maven groupId</th><td>$archive.group_id</td>
</tr>
<tr>
<th>Maven artifactId</th><td>$maveninfo.artifact_id</td>
<th>Maven artifactId</th><td>$archive.artifact_id</td>
</tr>
<tr>
<th>Maven version</th><td>$maveninfo.version</td>
<th>Maven version</th><td>$archive.version</td>
</tr>
#end if
<tr>
@ -43,10 +43,10 @@
</tr>
#if $wininfo
<tr>
<th>Platforms</th><td>$wininfo.platforms</td>
<th>Platforms</th><td>$archive.platforms</td>
</tr>
<tr>
<th>Flags</th><td>$wininfo.flags</td>
<th>Flags</th><td>$archive.flags</td>
</tr>
#end if
#if $builtInRoot

View file

@ -1293,8 +1293,12 @@ def archiveinfo(req, archiveID, fileOrder='name', fileStart=None, buildrootOrder
archive = server.getArchive(archiveID)
archive_type = server.getArchiveType(type_id=archive['type_id'])
build = server.getBuild(archive['build_id'])
maveninfo = server.getMavenArchive(archive['id'])
wininfo = server.getWinArchive(archive['id'])
maveninfo = False
if 'group_id' in archive:
maveninfo = True
wininfo = False
if 'relpath' in archive:
wininfo = True
builtInRoot = None
if archive['buildroot_id'] != None:
builtInRoot = server.getBuildroot(archive['buildroot_id'])