diff --git a/builder/kojid b/builder/kojid index 87729a3f..6f334d79 100755 --- a/builder/kojid +++ b/builder/kojid @@ -824,17 +824,6 @@ class BuildTask(BaseTaskHandler): raise koji.BuildError, "No matching arches were found" return archdict.keys() - def getRepo(self, tag): - """Get repo to use for builds""" - repo_info = self.session.getRepo(tag) - if not repo_info: - #wait for it - task_id = self.session.host.subtask(method='waitrepo', - arglist=[tag, None, None], - parent=self.id) - repo_info = self.wait(task_id)[task_id] - return repo_info - def runBuilds(self, srpm, build_tag, archlist, repo_id): self.logger.debug("Spawning jobs for arches: %r" % (archlist)) subtasks = {} diff --git a/hub/kojihub.py b/hub/kojihub.py index d5ad6d39..f5651f73 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3432,11 +3432,32 @@ def get_archive(archive_id, strict=False): filename: name of the archive (string) size: size of the archive (integer) md5sum: md5sum of the archive (string) + + If the archive is part of a Maven build, the following keys will be included: + group_id + artifact_id + version + If the archive is part of a Windows builds, the following keys will be included: + relpath + platforms + flags """ fields = ('id', 'type_id', 'build_id', 'buildroot_id', 'filename', 'size', 'md5sum') select = """SELECT %s FROM archiveinfo WHERE id = %%(archive_id)i""" % ', '.join(fields) - return _singleRow(select, locals(), fields, strict=strict) + archive = _singleRow(select, locals(), fields, strict=strict) + if not archive: + # strict is taken care of by _singleRow() + return None + maven_info = get_maven_archive(archive_id) + if maven_info: + del maven_info['archive_id'] + archive.update(maven_info) + win_info = get_win_archive(archive_id) + if win_info: + del win_info['archive_id'] + archive.update(win_info) + return archive def get_maven_archive(archive_id, strict=False): """ diff --git a/koji/__init__.py b/koji/__init__.py index 6fc0a757..8ce30990 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1394,17 +1394,24 @@ class PathInfo(object): """Return the relative path from the winbuild directory where the file identified by wininfo is located.""" filepath = wininfo['filename'] - if wininfo.get('relpath'): + if wininfo['relpath']: filepath = wininfo['relpath'] + '/' + filepath return filepath + def mavenfile(self, maveninfo): + """Return the relative path the file exists in the per-tag Maven repo""" + group_path = maveninfo['group_id'].replace('.', '/') + artifact_id = maveninfo['artifact_id'] + version = maveninfo['version'] + return "%(group_path)s/%(artifact_id)s/%(version)s" % locals() + def mavenrepo(self, build, maveninfo): """Return the directory where the Maven artifact exists in the per-tag Maven repo (/mnt/koji/repos/tag-name/repo-id/maven2/)""" group_path = maveninfo['group_id'].replace('.', '/') artifact_id = maveninfo['artifact_id'] version = maveninfo['version'] - return self.topdir + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s" % locals()) + return self.topdir + "/maven2/" + self.mavenfile(maveninfo) def rpm(self,rpminfo): """Return the path (relative to build_dir) where an rpm belongs""" diff --git a/koji/daemon.py b/koji/daemon.py index d46abb55..7dfad759 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -741,8 +741,7 @@ class TaskManager(object): self.logger.debug("available capacities for bin: %r" % bin_avail) median = bin_avail[(len(bin_avail)-1)/2] self.logger.debug("ours: %.2f, median: %.2f" % (our_avail, median)) - if our_avail < median: - self.logger.debug("Skipping - available capacity in lower half") + if not self.checkRelAvail(bin_avail, our_avail): #decline for now and give the upper half a chance return False #otherwise, we attempt to open the task @@ -753,6 +752,19 @@ class TaskManager(object): raise Exception, "Invalid task state reported by server" return False + def checkRelAvail(self, bin_avail, avail): + """ + Check our available capacity against the capacity of other hosts in this bin. + Return True if we should take a task, False otherwise. + """ + median = bin_avail[(len(bin_avail)-1)/2] + self.logger.debug("ours: %.2f, median: %.2f" % (avail, median)) + if avail >= median: + return True + else: + self.logger.debug("Skipping - available capacity in lower half") + return False + def _waitTask(self, task_id, pid=None): """Wait (nohang) on the task, return true if finished""" if pid is None: diff --git a/koji/tasks.py b/koji/tasks.py index e8dbb068..7063721a 100644 --- a/koji/tasks.py +++ b/koji/tasks.py @@ -320,6 +320,21 @@ class BaseTaskHandler(object): raise koji.BuildError, "host %s (%s) does not support any arches of tag %s (%s)" % \ (host['name'], ', '.join(host_arches), tag['name'], ', '.join(tag_arches)) + def getRepo(self, tag): + """ + Get the active repo for the given tag. If there is no repo available, + wait for a repo to be created. + """ + repo_info = self.session.getRepo(tag) + if not repo_info: + #wait for it + task_id = self.session.host.subtask(method='waitrepo', + arglist=[tag, None, None], + parent=self.id) + repo_info = self.wait(task_id)[task_id] + return repo_info + + class FakeTask(BaseTaskHandler): Methods = ['someMethod'] Foreground = True diff --git a/vm/kojikamid b/vm/kojikamid index 800f9922..3ab30550 100755 --- a/vm/kojikamid +++ b/vm/kojikamid @@ -304,7 +304,7 @@ class WindowsBuild(object): self.release = None self.description = None self.platform = None - self.buildrequires = {} + self.buildrequires = [] self.provides = [] self.shell = None self.execute = [] @@ -360,8 +360,22 @@ class WindowsBuild(object): self.platform = conf.get('building', 'platform') # buildrequires and provides are multi-valued (space-separated) for br in conf.get('building', 'buildrequires').split(): + # buildrequires is a space-separated list + # each item in the list is in the format: + # pkgname[:opt1:opt2=val2:...] + # the options are put into a dict + # if the option has no =val, the value in the dict will be None if br: - self.buildrequires[br] = {} + br = br.split(':') + bropts = {} + for opt in br[1:]: + if '=' in opt: + key, val = opt.split('=', 1) + else: + key = opt + val = None + bropts[key] = val + self.buildrequires.append((br[0], bropts)) for prov in conf.get('building', 'provides').split(): if prov: self.provides.append(prov) @@ -389,12 +403,9 @@ class WindowsBuild(object): self.output[filename] = metadata self.logs.extend([e.strip() for e in conf.get('files', 'logs').split('\n') if e]) - def fetchFile(self, basedir, buildinfo, fileinfo, type=None): + def fetchFile(self, basedir, buildinfo, fileinfo, type): """Download the file from buildreq, at filepath, into the basedir""" - if type == 'win': - destpath = os.path.join(basedir, fileinfo['relpath'], fileinfo['filename']) - else: - raise BuildError, 'unsupported file type: %s' % type + destpath = os.path.join(basedir, fileinfo['localpath']) ensuredir(os.path.dirname(destpath)) destfile = file(destpath, 'w') offset = 0 @@ -410,28 +421,28 @@ class WindowsBuild(object): checksum.update(data) destfile.close() digest = checksum.hexdigest() - if digest != fileinfo['md5sum']: + # rpms don't have a md5sum in the fileinfo, but check it for everything else + if ('md5sum' in fileinfo) and (digest != fileinfo['md5sum']): raise BuildError, 'md5 checksum validation failed for %s, %s (computed) != %s (provided)' % \ (destpath, digest, fileinfo['md5sum']) self.logger.info('Retrieved %s (%s bytes, md5: %s)', destpath, offset, digest) def fetchBuildReqs(self): """Retrieve buildrequires listed in the spec file""" - for buildreq, brinfo in self.buildrequires.items(): + for buildreq, brinfo in self.buildrequires: + # if no type is specified in the options, default to win + brtype = brinfo.get('type', 'win') buildinfo = self.server.getLatestBuild(self.build_tag, buildreq, - self.task_opts.get('event_id')) - br_dir = os.path.join(self.buildreq_dir, buildreq) + self.task_opts.get('repo_id')) + br_dir = os.path.join(self.buildreq_dir, buildreq, brtype) ensuredir(br_dir) brinfo['dir'] = br_dir brfiles = [] brinfo['files'] = brfiles - buildfiles = self.server.getFileList(buildinfo['id'], 'win') + buildfiles = self.server.getFileList(buildinfo['id'], brtype, brinfo) for fileinfo in buildfiles: - self.fetchFile(br_dir, buildinfo, fileinfo, 'win') - if fileinfo['relpath']: - brfiles.append(os.path.join(fileinfo['relpath'], fileinfo['filename'])) - else: - brfiles.append(fileinfo['filename']) + self.fetchFile(br_dir, buildinfo, fileinfo, brtype) + brfiles.append(fileinfo['localpath']) def build(self): if self.shell in ('cmd', 'cmd.exe'): @@ -455,14 +466,22 @@ class WindowsBuild(object): """Do the build: run the execute line(s) with cmd.exe""" tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp', suffix='.bat', dir='/cygdrive/c/Windows/Temp') script = os.fdopen(tmpfd, 'w') - for buildreq, brinfo in self.buildrequires.items(): + for buildreq, brinfo in self.buildrequires: buildreq = self.varname(buildreq) ret, output = run(['cygpath', '-wa', brinfo['dir']], log=False, fatal=True) br_dir = output.strip() - script.write('set %s_dir=%s\r\n' % (buildreq, br_dir)) files = ' '.join(brinfo['files']) files.replace('/', '\\') - script.write('set %s_files=%s\r\n' % (buildreq, files)) + if brinfo.get('type'): + # if the spec file qualifies the buildreq with a type, + # the env. var is named buildreq_type_{dir,files} + script.write('set %s_%s_dir=%s\r\n' % (buildreq, brinfo['type'], br_dir)) + script.write('set %s_%s_files=%s\r\n' % (buildreq, brinfo['type'], files)) + else: + # otherwise it's just buildreq_{dir,files} + script.write('set %s_dir=%s\r\n' % (buildreq, br_dir)) + script.write('set %s_files=%s\r\n' % (buildreq, files)) + script.write('\r\n') for cmd in self.execute: script.write(cmd) script.write('\r\n') @@ -476,10 +495,14 @@ class WindowsBuild(object): """Do the build: run the execute line(s) with bash""" tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp.', dir='/tmp') script = os.fdopen(tmpfd, 'w') - for buildreq, brinfo in self.buildrequires.items(): + for buildreq, brinfo in self.buildrequires: buildreq = self.varname(buildreq) - script.write("export %s_dir='%s'\n" % (buildreq, brinfo['dir'])) - script.write("export %s_files='" % buildreq) + if brinfo.get('type'): + script.write("export %s_%s_dir='%s'\n" % (buildreq, brinfo['type'], brinfo['dir'])) + script.write("export %s_%s_files='" % (buildreq, brinfo['type'])) + else: + script.write("export %s_dir='%s'\n" % (buildreq, brinfo['dir'])) + script.write("export %s_files='" % buildreq) for filename in brinfo['files']: script.write(filename) script.write('\n') @@ -664,19 +687,26 @@ def incremental_upload(server, handler): else: time.sleep(1) -def flunk(server, logfile): +def flunk(server, handler): """do the right thing when a build fails""" global logfd logging.getLogger('koji.vm').error('error running build', exc_info=True) tb = ''.join(traceback.format_exception(*sys.exc_info())) + handler.active = False if server is not None: try: logfd.flush() - upload_file(server, os.path.dirname(logfile), - os.path.basename(logfile)) + upload_file(server, os.path.dirname(handler.baseFilename), + os.path.basename(handler.baseFilename)) except: - pass - server.failTask(tb) + log_local('error calling upload_file()') + while True: + try: + # this is the very last thing we do, keep trying as long as we can + server.failTask(tb) + break + except: + log_local('error calling server.failTask()') sys.exit(1) logfd = None @@ -715,6 +745,7 @@ if __name__ == '__main__': thread = threading.Thread(target=incremental_upload, args=(server, handler)) + thread.daemon = True thread.start() build = WindowsBuild(server) @@ -732,5 +763,5 @@ if __name__ == '__main__': server.closeTask(results) except: - flunk(server, logfile) + flunk(server, handler) sys.exit(0) diff --git a/vm/kojivmd b/vm/kojivmd index babfacbe..3f912227 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -41,6 +41,7 @@ import threading import base64 import pwd import urlgrabber +import fnmatch from ConfigParser import ConfigParser from optparse import OptionParser try: @@ -301,6 +302,7 @@ class WinBuildTask(BaseTaskHandler): repo_id = opts.get('repo_id') if repo_id: repo_info = session.repoInfo(repo_id) + event_id = repo_info['create_event'] if not repo_info: raise koji.BuildError, 'invalid repo ID: %s' % repo_id policy_data = { @@ -314,11 +316,12 @@ class WinBuildTask(BaseTaskHandler): if not opts.get('skip_tag'): policy_data['tag'] = dest_tag['id'] self.session.host.assertPolicy('build_from_repo_id', policy_data) - event_id = repo_info['create_event'] else: - event_id = self.session.getLastEvent()['id'] + repo_info = self.getRepo(build_tag['id']) + repo_id = repo_info['id'] + event_id = None - subopts['event_id'] = event_id + subopts['repo_id'] = repo_id task_opts = koji.util.dslice(opts, ['timeout', 'cpus', 'mem'], strict=False) task_id = self.session.host.subtask(method='vmExec', @@ -463,39 +466,110 @@ class VMExecTask(BaseTaskHandler): """ return self.task_info - def getLatestBuild(self, tag, package, event=None): + def getLatestBuild(self, tag, package, repo_id): """ Get information about the latest build of package "package" in tag "tag". """ - builds = self.session.getLatestBuilds(tag, package=package, event=event) + repo_info = self.session.repoInfo(repo_id, strict=True) + builds = self.session.getLatestBuilds(tag, package=package, + event=repo_info['create_event']) if not builds: raise koji.BuildError, 'no build of package %s in tag %s' % (package, tag) - return builds[0] + build = builds[0] + maven_build = self.session.getMavenBuild(build['id']) + if maven_build: + del maven_build['build_id'] + build.update(maven_build) + win_build = self.session.getWinBuild(build['id']) + if win_build: + del win_build['build_id'] + build.update(win_build) + return build - def getFileList(self, buildID, type=None): + def getFileList(self, buildID, type, typeopts): """ - Get the file list for the latest build of the package "package" in tag "tag". - If type is specified, include the extended information for archives of that type. + Get the list of files of "type" for the latest build of the package "package" in tag "tag". + typeopts is a dict that is used to filter the file list. + typeopts is checked for: + patterns: comma-separated list of path/filename patterns (as used by fnmatch) + to filter the results with + If type is 'rpm', typeopts is checked for: + arches: comma-separated list of arches to include in output + If type is 'maven', typeopts is checked for: + group_ids: Maven group IDs to include in the output + artifact_ids: Maven artifact IDs to include in the output + versions: Maven versions to include in the output + If type is 'win', typeopts is checked for: + platforms: comma-separated list of platforms + flags: comma-separated list of flags """ - return self.session.listArchives(buildID=buildID, type=type) + if not typeopts: + typeopts = {} + if type == 'rpm': + arches = None + if typeopts.get('arches'): + arches = typeopts['arches'].split(',') + files = self.session.listRPMs(buildID=buildID, arches=arches) + else: + files = self.session.listArchives(buildID=buildID, type=type) + for fileinfo in files: + if type == 'rpm': + filepath = koji.pathinfo.rpm(fileinfo) + elif type == 'maven': + filepath = koji.pathinfo.mavenfile(fileinfo) + elif type == 'win': + filepath = koji.pathinfo.winfile(fileinfo) + else: + # XXX support other file types when available + filepath = fileinfo['filename'] + fileinfo['localpath'] = filepath + if typeopts.get('patterns'): + to_filter = files + files = [] + patterns = typeopts['patterns'].split(',') + for fileinfo in to_filter: + for pattern in patterns: + if fnmatch.fnmatch(fileinfo['localpath'], pattern): + files.append(fileinfo) + break + if type == 'maven': + if typeopts.get('group_ids'): + group_ids = typeopts['group_ids'].split(',') + files = [f for f in files if f['group_id'] in group_ids] + if typeopts.get('artifact_ids'): + artifact_ids = typeopts['artifact_ids'].split(',') + files = [f for f in files if f['artifact_id'] in artifact_ids] + if typeopts.get('versions'): + versions = typeopts['versions'].split(',') + files = [f for f in files if f['version'] in versions] + if type == 'win': + if typeopts.get('platforms'): + platforms = typeopts['platforms'].split(',') + files = [f for f in files if set(f['platforms'].split()).intersection(platforms)] + if typeopts.get('flags'): + flags = typeopts['flags'].split(',') + files = [f for f in files if set(f['flags'].split()).intersection(flags)] + return files - def localCache(self, buildinfo, fileinfo, type=None): + def localCache(self, buildinfo, fileinfo, type): """ Access a file in the local cache. If the file does not exist, it's downloaded from the server. Returns an open file object. """ - local_pi = koji.PathInfo(self.buildreq_dir) - if type == 'win': - localpath = os.path.join(local_pi.winbuild(buildinfo), - fileinfo['relpath'], fileinfo['filename']) - else: - raise koji.BuildError, 'unsupported file type: %s' % type - + # fileinfo['localpath'] is set by getFileList() + localpath = os.path.join(self.buildreq_dir, type, fileinfo['localpath']) if not os.path.isfile(localpath): remote_pi = koji.PathInfo(self.options.topurl) - if type == 'win': + if type == 'rpm': + remote_url = remote_pi.build(buildinfo) + '/' + \ + remote_pi.rpm(fileinfo) + elif type == 'maven': + # the relevant Maven info is included in the buildinfo + remote_url = remote_pi.mavenbuild(buildinfo, buildinfo) + '/' + \ + fileinfo['filename'] + elif type == 'win': remote_url = remote_pi.winbuild(buildinfo) + '/' + \ - fileinfo['relpath'] + '/' + fileinfo['filename'] + remote_pi.winfile(fileinfo) else: raise koji.BuildError, 'unsupported file type: %s' % type koji.ensuredir(os.path.dirname(localpath)) @@ -503,14 +577,14 @@ class VMExecTask(BaseTaskHandler): return file(localpath, 'r') - def getFile(self, buildinfo, archiveinfo, offset, length, type=None): + def getFile(self, buildinfo, archiveinfo, offset, length, type): """ Get the contents of the file indicated by fileinfo, returning a maximum of "length" bytes starting at "offset". Contents are returned base64-encoded. """ offset = int(offset) length = int(length) - fileobj = self.localCache(buildinfo, archiveinfo, type=type) + fileobj = self.localCache(buildinfo, archiveinfo, type) try: fileobj.seek(offset) data = fileobj.read(length) @@ -782,6 +856,13 @@ class VMTaskManager(TaskManager): """See if we have enough space to accept another job""" return self.checkDisk() and self.checkMem() + def checkRelAvail(self, bin_avail, avail): + """ + Always return True, since we may be the only daemon with access + to the VM required to process this task. + """ + return True + def takeTask(self, task): """ Verify that this builder can handle the task before claiming it. diff --git a/www/kojiweb/archiveinfo.chtml b/www/kojiweb/archiveinfo.chtml index c0f46184..6712d30b 100644 --- a/www/kojiweb/archiveinfo.chtml +++ b/www/kojiweb/archiveinfo.chtml @@ -26,13 +26,13 @@ #if $maveninfo - Maven groupId$maveninfo.group_id + Maven groupId$archive.group_id - Maven artifactId$maveninfo.artifact_id + Maven artifactId$archive.artifact_id - Maven version$maveninfo.version + Maven version$archive.version #end if @@ -43,10 +43,10 @@ #if $wininfo - Platforms$wininfo.platforms + Platforms$archive.platforms - Flags$wininfo.flags + Flags$archive.flags #end if #if $builtInRoot diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 40683e03..84216c77 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -1293,8 +1293,12 @@ def archiveinfo(req, archiveID, fileOrder='name', fileStart=None, buildrootOrder archive = server.getArchive(archiveID) archive_type = server.getArchiveType(type_id=archive['type_id']) build = server.getBuild(archive['build_id']) - maveninfo = server.getMavenArchive(archive['id']) - wininfo = server.getWinArchive(archive['id']) + maveninfo = False + if 'group_id' in archive: + maveninfo = True + wininfo = False + if 'relpath' in archive: + wininfo = True builtInRoot = None if archive['buildroot_id'] != None: builtInRoot = server.getBuildroot(archive['buildroot_id'])