diff --git a/hub/kojihub.py b/hub/kojihub.py index 9574f02c..d4093b0f 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -3438,7 +3438,7 @@ def get_archive(archive_id, strict=False): WHERE id = %%(archive_id)i""" % ', '.join(fields) return _singleRow(select, locals(), fields, strict=strict) -def get_maven_archive(archive_id): +def get_maven_archive(archive_id, strict=False): """ Retrieve Maven-specific information about an archive. Returns a map containing the following keys: @@ -3451,9 +3451,9 @@ def get_maven_archive(archive_id): fields = ('archive_id', 'group_id', 'artifact_id', 'version') select = """SELECT %s FROM maven_archives WHERE archive_id = %%(archive_id)i""" % ', '.join(fields) - return _singleRow(select, locals(), fields) + return _singleRow(select, locals(), fields, strict=strict) -def get_win_archive(archive_id): +def get_win_archive(archive_id, strict=False): """ Retrieve Windows-specific information about an archive. Returns a map containing the following keys: @@ -3466,7 +3466,7 @@ def get_win_archive(archive_id): fields = ('archive_id', 'relpath', 'platforms', 'flags') select = """SELECT %s FROM win_archives WHERE archive_id = %%(archive_id)i""" % ', '.join(fields) - return _singleRow(select, locals(), fields) + return _singleRow(select, locals(), fields, strict=strict) def _get_zipfile_list(archive_id, zippath): """ @@ -4397,6 +4397,7 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None, destp _import_archive_file(filepath, mavendir) _generate_maven_metadata(maveninfo, mavendir) elif type == 'win': + wininfo = get_win_build(buildinfo, strict=True) insert = InsertProcessor('win_archives') insert.set(archive_id=archive_id) insert.set(relpath=destpath) @@ -4404,8 +4405,7 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None, destp if typeInfo['flags']: insert.set(flags=' '.join(typeInfo['flags'])) insert.execute() - wininfo = get_win_build(buildinfo, strict=True) - destdir = koji.pathinfo.winbuild(buildinfo, wininfo) + destdir = koji.pathinfo.winbuild(buildinfo) if destpath: destdir = os.path.join(destdir, destpath) _import_archive_file(filepath, destdir) diff --git a/koji/__init__.py b/koji/__init__.py index 8b89ad7d..58b80922 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1386,7 +1386,7 @@ class PathInfo(object): release = build['release'] return self.topdir + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s/%(release)s" % locals()) - def winbuild(self, build, wininfo): + def winbuild(self, build): """Return the directory where the Windows build exists""" return self.build(build) + '/win' diff --git a/vm/kojikamid b/vm/kojikamid index 2b811eae..40a1e37a 100755 --- a/vm/kojikamid +++ b/vm/kojikamid @@ -38,6 +38,7 @@ import xmlrpclib import base64 import hashlib import traceback +import base64 MANAGER_PORT = 7000 @@ -294,6 +295,8 @@ class WindowsBuild(object): self.task_opts = {} self.workdir = '/tmp/build' ensuredir(self.workdir) + self.buildreq_dir = os.path.join(self.workdir, 'buildreqs') + ensuredir(self.buildreq_dir) self.source_dir = None self.spec_dir = None self.logfile = logfile @@ -304,7 +307,7 @@ class WindowsBuild(object): self.release = None self.description = None self.platform = None - self.buildrequires = [] + self.buildrequires = {} self.provides = [] self.execute = [] self.output = {} @@ -360,8 +363,12 @@ class WindowsBuild(object): # [building] section self.platform = conf.get('building', 'platform') # buildrequires and provides are multi-valued (space-separated) - for entry in ('buildrequires', 'provides'): - getattr(self, entry).extend([e for e in conf.get('building', entry).split() if e]) + for br in conf.get('building', 'buildrequires').split(): + if br: + self.buildrequires[br] = {} + for prov in conf.get('building', 'provides').split(): + if prov: + self.provides.append(prov) # execute is multi-valued (newline-separated) self.execute.extend([e.strip() for e in conf.get('building', 'execute').split('\n') if e]) @@ -380,15 +387,49 @@ class WindowsBuild(object): self.output[filename] = metadata self.logs.extend([e.strip() for e in conf.get('files', 'logs').split('\n') if e]) + def fetchFile(self, basedir, buildinfo, fileinfo, type=None): + """Download the file from buildreq, at filepath, into the basedir""" + destpath = os.path.join(basedir, fileinfo['relpath'], fileinfo['filename']) + ensuredir(os.path.dirname(destpath)) + destfile = file(destpath, 'w') + offset = 0 + while True: + encoded = self.server.getFile(buildinfo, fileinfo, encode_int(offset), 1048576, type) + if not encoded: + break + data = base64.b64decode(encoded) + del encoded + destfile.write(data) + offset += len(data) + destfile.close() + self.logfile.write('Retrieved %s (%s bytes)\n' % (destpath, offset)) + def fetchBuildReqs(self): """Retrieve buildrequires listed in the spec file""" - for br in self.buildrequires: - pass + for buildreq, brinfo in self.buildrequires.items(): + buildinfo = self.server.getLatestBuild(self.build_tag, buildreq, + self.task_opts.get('event_id')) + br_dir = os.path.join(self.buildreq_dir, buildreq) + ensuredir(br_dir) + brinfo['dir'] = br_dir + brfiles = [] + brinfo['files'] = brfiles + buildfiles = self.server.getFileList(buildinfo['id'], 'win') + for fileinfo in buildfiles: + self.fetchFile(br_dir, buildinfo, fileinfo, 'win') + brfiles.append(os.path.join(fileinfo['relpath'], fileinfo['filename'])) def build(self): """Do the build: run the execute line(s)""" tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp.', dir='/tmp') script = os.fdopen(tmpfd, 'w') + for buildreq, brinfo in self.buildrequires.items(): + script.write("export %s_dir='%s'\n" % (buildreq, brinfo['dir'])) + script.write("export %s_files='" % buildreq) + for filename in brinfo['files']: + script.write(filename) + script.write('\n') + script.write("'\n\n") for cmd in self.execute: script.write(cmd) script.write('\n') @@ -479,6 +520,12 @@ def find_net_info(): gateway = None return macaddr, gateway +def encode_int(n): + """If n is too large for a 32bit signed, convert it to a string""" + if n <= 2147483647: + return n + return str(n) + def upload_file(server, prefix, path): """upload a single file to the vmd""" fobj = file(os.path.join(prefix, path), 'r') @@ -489,7 +536,7 @@ def upload_file(server, prefix, path): if not data: break encoded = base64.b64encode(data) - server.upload(path, offset, encoded) + server.upload(path, encode_int(offset), encoded) offset += len(data) sum.update(data) fobj.close() diff --git a/vm/kojivmd b/vm/kojivmd index 6e6b9a9a..70a8cd81 100755 --- a/vm/kojivmd +++ b/vm/kojivmd @@ -40,6 +40,7 @@ import SimpleXMLRPCServer import threading import base64 import pwd +import urlgrabber from ConfigParser import ConfigParser from optparse import OptionParser try: @@ -107,6 +108,7 @@ def get_options(): 'vmuser': 'qemu', 'admin_emails': None, 'workdir': '/tmp/koji', + 'topurl': '', 'imagedir': '/var/lib/libvirt/images', 'pluginpath': '/usr/lib/koji-vm-plugins', 'privaddr': '192.168.122.1', @@ -221,7 +223,11 @@ class DaemonXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer): allow_reuse_address = True def __init__(self, addr, port): - SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False) + if sys.version_info[:2] <= (2, 4): + SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False) + else: + SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False, + allow_none=True) self.logger = logging.getLogger('koji.vm.' + self.__class__.__name__) self.socket.settimeout(5) self.active = True @@ -245,6 +251,28 @@ class DaemonXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer): except: self.logger.error('Error handling requests', exc_info=True) + if sys.version_info[:2] <= (2, 4): + # Copy and paste from SimpleXMLRPCServer, with the addition of passing + # allow_none=True to xmlrpclib.dumps() + def _marshaled_dispatch(self, data, dispatch_method = None): + params, method = xmlrpclib.loads(data) + try: + if dispatch_method is not None: + response = dispatch_method(method, params) + else: + response = self._dispatch(method, params) + response = (response,) + response = xmlrpclib.dumps(response, methodresponse=1, allow_none=True) + except xmlrpclib.Fault, fault: + response = xmlrpclib.dumps(fault) + except: + # report exception back to server + response = xmlrpclib.dumps( + xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value)) + ) + return response + + class TaskXMLRPCServer(DaemonXMLRPCServer): def __init__(self, addr, port, task_handler): @@ -252,6 +280,9 @@ class TaskXMLRPCServer(DaemonXMLRPCServer): self.register_function(task_handler.getTaskInfo) self.register_function(task_handler.closeTask) self.register_function(task_handler.failTask) + self.register_function(task_handler.getLatestBuild) + self.register_function(task_handler.getFileList) + self.register_function(task_handler.getFile) self.register_function(task_handler.upload) self.register_function(task_handler.verifyChecksum) @@ -363,6 +394,10 @@ class VMExecTask(BaseTaskHandler): self.port = None self.server = None self.task_info = None + self.buildreq_dir = os.path.join(self.workdir, 'buildreqs') + koji.ensuredir(self.buildreq_dir) + self.output_dir = os.path.join(self.workdir, 'output') + koji.ensuredir(self.output_dir) self.output = None self.success = None @@ -442,9 +477,66 @@ class VMExecTask(BaseTaskHandler): """ return self.task_info + def getLatestBuild(self, tag, package, event=None): + """ + Get information about the latest build of package "package" in tag "tag". + """ + builds = self.session.getLatestBuilds(tag, package=package, event=event) + if not builds: + raise koji.BuildError, 'no build of package %s in tag %s' % (package, tag) + return builds[0] + + def getFileList(self, buildID, type=None): + """ + Get the file list for the latest build of the package "package" in tag "tag". + If type is specified, include the extended information for archives of that type. + """ + return self.session.listArchives(buildID=buildID, type=type) + + def localCache(self, buildinfo, fileinfo, type=None): + """ + Access a file in the local cache. If the file does not exist, it's downloaded + from the server. Returns an open file object. + """ + local_pi = koji.PathInfo(self.buildreq_dir) + if type == 'win': + localpath = os.path.join(local_pi.winbuild(buildinfo), + fileinfo['relpath'], fileinfo['filename']) + else: + raise koji.BuildError, 'unsupported file type: %s' % type + + if not os.path.isfile(localpath): + remote_pi = koji.PathInfo(self.options.topurl) + if type == 'win': + remote_url = remote_pi.winbuild(buildinfo) + '/' + \ + fileinfo['relpath'] + '/' + fileinfo['filename'] + else: + raise koji.BuildError, 'unsupported file type: %s' % type + koji.ensuredir(os.path.dirname(localpath)) + urlgrabber.urlgrab(remote_url, filename=localpath) + + return file(localpath, 'r') + + def getFile(self, buildinfo, archiveinfo, offset, length, type=None): + """ + Get the contents of the file indicated by fileinfo, returning a maximum of + "length" bytes starting at "offset". Contents are returned base64-encoded. + """ + offset = int(offset) + length = int(length) + fileobj = self.localCache(buildinfo, archiveinfo, type=type) + try: + fileobj.seek(offset) + data = fileobj.read(length) + encoded = base64.b64encode(data) + del data + return encoded + finally: + fileobj.close() + def upload(self, path, offset, contents): - local_path = os.path.abspath(os.path.join(self.workdir, path)) - if not local_path.startswith(self.workdir): + local_path = os.path.abspath(os.path.join(self.output_dir, path)) + if not local_path.startswith(self.output_dir): raise koji.BuildError, 'invalid upload path: %s' % path koji.ensuredir(os.path.dirname(local_path)) # accept offset as a str to avoid problems with files larger than 2**32 @@ -468,8 +560,8 @@ class VMExecTask(BaseTaskHandler): return len(data) def verifyChecksum(self, path, checksum, algo='sha1'): - local_path = os.path.abspath(os.path.join(self.workdir, path)) - if not local_path.startswith(self.workdir): + local_path = os.path.abspath(os.path.join(self.output_dir, path)) + if not local_path.startswith(self.output_dir): raise koji.BuildError, 'invalid path: %s' % path if not os.path.isfile(local_path): raise koji.BuildError, '%s does not exist' % local_path @@ -578,7 +670,7 @@ class VMExecTask(BaseTaskHandler): else: vm.destroy() self.server.server_close() - self.uploadTree(self.workdir) + self.uploadTree(self.output_dir) if self.success: return self.output else: diff --git a/vm/kojivmd.conf b/vm/kojivmd.conf index d4b1a4a8..9121df83 100644 --- a/vm/kojivmd.conf +++ b/vm/kojivmd.conf @@ -14,6 +14,9 @@ ; The directory root for temporary storage ; workdir=/tmp/koji +; The url where the Koji root directory (/mnt/koji) can be accessed +topurl=http://koji.example.com/kojiroot + ; The URL for the xmlrpc server server=http://hub.example.com/kojihub