particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

support for retrieving buildrequires

Browse source
This commit is contained in:
Mike Bonnet 2010-07-26 11:58:51 -04:00
parent ff296efd25
commit fa5c7722cd
5 changed files with 161 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

12
hub/kojihub.py
View file

@ -3438,7 +3438,7 @@ def get_archive(archive_id, strict=False):
WHERE id = %%(archive_id)i""" % ', '.join(fields)
return _singleRow(select, locals(), fields, strict=strict)
def get_maven_archive(archive_id):
def get_maven_archive(archive_id, strict=False):
"""
Retrieve Maven-specific information about an archive.
Returns a map containing the following keys:
@ -3451,9 +3451,9 @@ def get_maven_archive(archive_id):
fields = ('archive_id', 'group_id', 'artifact_id', 'version')
select = """SELECT %s FROM maven_archives
WHERE archive_id = %%(archive_id)i""" % ', '.join(fields)
return _singleRow(select, locals(), fields)
return _singleRow(select, locals(), fields, strict=strict)
def get_win_archive(archive_id):
def get_win_archive(archive_id, strict=False):
"""
Retrieve Windows-specific information about an archive.
Returns a map containing the following keys:
@ -3466,7 +3466,7 @@ def get_win_archive(archive_id):
fields = ('archive_id', 'relpath', 'platforms', 'flags')
select = """SELECT %s FROM win_archives
WHERE archive_id = %%(archive_id)i""" % ', '.join(fields)
return _singleRow(select, locals(), fields)
return _singleRow(select, locals(), fields, strict=strict)
def _get_zipfile_list(archive_id, zippath):
"""
@ -4397,6 +4397,7 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None, destp
_import_archive_file(filepath, mavendir)
_generate_maven_metadata(maveninfo, mavendir)
elif type == 'win':
wininfo = get_win_build(buildinfo, strict=True)
insert = InsertProcessor('win_archives')
insert.set(archive_id=archive_id)
insert.set(relpath=destpath)
@ -4404,8 +4405,7 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None, destp
if typeInfo['flags']:
insert.set(flags=' '.join(typeInfo['flags']))
insert.execute()
wininfo = get_win_build(buildinfo, strict=True)
destdir = koji.pathinfo.winbuild(buildinfo, wininfo)
destdir = koji.pathinfo.winbuild(buildinfo)
if destpath:
destdir = os.path.join(destdir, destpath)
_import_archive_file(filepath, destdir)

2
koji/__init__.py
View file

@ -1386,7 +1386,7 @@ class PathInfo(object):
release = build['release']
return self.topdir + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s/%(release)s" % locals())
def winbuild(self, build, wininfo):
def winbuild(self, build):
"""Return the directory where the Windows build exists"""
return self.build(build) + '/win'

59
vm/kojikamid
View file

@ -38,6 +38,7 @@ import xmlrpclib
import base64
import hashlib
import traceback
import base64
MANAGER_PORT = 7000
@ -294,6 +295,8 @@ class WindowsBuild(object):
self.task_opts = {}
self.workdir = '/tmp/build'
ensuredir(self.workdir)
self.buildreq_dir = os.path.join(self.workdir, 'buildreqs')
ensuredir(self.buildreq_dir)
self.source_dir = None
self.spec_dir = None
self.logfile = logfile
@ -304,7 +307,7 @@ class WindowsBuild(object):
self.release = None
self.description = None
self.platform = None
self.buildrequires = []
self.buildrequires = {}
self.provides = []
self.execute = []
self.output = {}
@ -360,8 +363,12 @@ class WindowsBuild(object):
# [building] section
self.platform = conf.get('building', 'platform')
# buildrequires and provides are multi-valued (space-separated)
for entry in ('buildrequires', 'provides'):
getattr(self, entry).extend([e for e in conf.get('building', entry).split() if e])
for br in conf.get('building', 'buildrequires').split():
if br:
self.buildrequires[br] = {}
for prov in conf.get('building', 'provides').split():
if prov:
self.provides.append(prov)
# execute is multi-valued (newline-separated)
self.execute.extend([e.strip() for e in conf.get('building', 'execute').split('\n') if e])
@ -380,15 +387,49 @@ class WindowsBuild(object):
self.output[filename] = metadata
self.logs.extend([e.strip() for e in conf.get('files', 'logs').split('\n') if e])
def fetchFile(self, basedir, buildinfo, fileinfo, type=None):
"""Download the file from buildreq, at filepath, into the basedir"""
destpath = os.path.join(basedir, fileinfo['relpath'], fileinfo['filename'])
ensuredir(os.path.dirname(destpath))
destfile = file(destpath, 'w')
offset = 0
while True:
encoded = self.server.getFile(buildinfo, fileinfo, encode_int(offset), 1048576, type)
if not encoded:
break
data = base64.b64decode(encoded)
del encoded
destfile.write(data)
offset += len(data)
destfile.close()
self.logfile.write('Retrieved %s (%s bytes)\n' % (destpath, offset))
def fetchBuildReqs(self):
"""Retrieve buildrequires listed in the spec file"""
for br in self.buildrequires:
pass
for buildreq, brinfo in self.buildrequires.items():
buildinfo = self.server.getLatestBuild(self.build_tag, buildreq,
self.task_opts.get('event_id'))
br_dir = os.path.join(self.buildreq_dir, buildreq)
ensuredir(br_dir)
brinfo['dir'] = br_dir
brfiles = []
brinfo['files'] = brfiles
buildfiles = self.server.getFileList(buildinfo['id'], 'win')
for fileinfo in buildfiles:
self.fetchFile(br_dir, buildinfo, fileinfo, 'win')
brfiles.append(os.path.join(fileinfo['relpath'], fileinfo['filename']))
def build(self):
"""Do the build: run the execute line(s)"""
tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp.', dir='/tmp')
script = os.fdopen(tmpfd, 'w')
for buildreq, brinfo in self.buildrequires.items():
script.write("export %s_dir='%s'\n" % (buildreq, brinfo['dir']))
script.write("export %s_files='" % buildreq)
for filename in brinfo['files']:
script.write(filename)
script.write('\n')
script.write("'\n\n")
for cmd in self.execute:
script.write(cmd)
script.write('\n')
@ -479,6 +520,12 @@ def find_net_info():
gateway = None
return macaddr, gateway
def encode_int(n):
"""If n is too large for a 32bit signed, convert it to a string"""
if n <= 2147483647:
return n
return str(n)
def upload_file(server, prefix, path):
"""upload a single file to the vmd"""
fobj = file(os.path.join(prefix, path), 'r')
@ -489,7 +536,7 @@ def upload_file(server, prefix, path):
if not data:
break
encoded = base64.b64encode(data)
server.upload(path, offset, encoded)
server.upload(path, encode_int(offset), encoded)
offset += len(data)
sum.update(data)
fobj.close()

104
vm/kojivmd
View file

@ -40,6 +40,7 @@ import SimpleXMLRPCServer
import threading
import base64
import pwd
import urlgrabber
from ConfigParser import ConfigParser
from optparse import OptionParser
try:
@ -107,6 +108,7 @@ def get_options():
'vmuser': 'qemu',
'admin_emails': None,
'workdir': '/tmp/koji',
'topurl': '',
'imagedir': '/var/lib/libvirt/images',
'pluginpath': '/usr/lib/koji-vm-plugins',
'privaddr': '192.168.122.1',
@ -221,7 +223,11 @@ class DaemonXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer):
allow_reuse_address = True
def __init__(self, addr, port):
SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False)
if sys.version_info[:2] <= (2, 4):
SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False)
else:
SimpleXMLRPCServer.SimpleXMLRPCServer.__init__(self, (addr, port), logRequests=False,
allow_none=True)
self.logger = logging.getLogger('koji.vm.' + self.__class__.__name__)
self.socket.settimeout(5)
self.active = True
@ -245,6 +251,28 @@ class DaemonXMLRPCServer(SimpleXMLRPCServer.SimpleXMLRPCServer):
except:
self.logger.error('Error handling requests', exc_info=True)
if sys.version_info[:2] <= (2, 4):
# Copy and paste from SimpleXMLRPCServer, with the addition of passing
# allow_none=True to xmlrpclib.dumps()
def _marshaled_dispatch(self, data, dispatch_method = None):
params, method = xmlrpclib.loads(data)
try:
if dispatch_method is not None:
response = dispatch_method(method, params)
else:
response = self._dispatch(method, params)
response = (response,)
response = xmlrpclib.dumps(response, methodresponse=1, allow_none=True)
except xmlrpclib.Fault, fault:
response = xmlrpclib.dumps(fault)
except:
# report exception back to server
response = xmlrpclib.dumps(
xmlrpclib.Fault(1, "%s:%s" % (sys.exc_type, sys.exc_value))
)
return response
class TaskXMLRPCServer(DaemonXMLRPCServer):
def __init__(self, addr, port, task_handler):
@ -252,6 +280,9 @@ class TaskXMLRPCServer(DaemonXMLRPCServer):
self.register_function(task_handler.getTaskInfo)
self.register_function(task_handler.closeTask)
self.register_function(task_handler.failTask)
self.register_function(task_handler.getLatestBuild)
self.register_function(task_handler.getFileList)
self.register_function(task_handler.getFile)
self.register_function(task_handler.upload)
self.register_function(task_handler.verifyChecksum)
@ -363,6 +394,10 @@ class VMExecTask(BaseTaskHandler):
self.port = None
self.server = None
self.task_info = None
self.buildreq_dir = os.path.join(self.workdir, 'buildreqs')
koji.ensuredir(self.buildreq_dir)
self.output_dir = os.path.join(self.workdir, 'output')
koji.ensuredir(self.output_dir)
self.output = None
self.success = None
@ -442,9 +477,66 @@ class VMExecTask(BaseTaskHandler):
"""
return self.task_info
def getLatestBuild(self, tag, package, event=None):
"""
Get information about the latest build of package "package" in tag "tag".
"""
builds = self.session.getLatestBuilds(tag, package=package, event=event)
if not builds:
raise koji.BuildError, 'no build of package %s in tag %s' % (package, tag)
return builds[0]
def getFileList(self, buildID, type=None):
"""
Get the file list for the latest build of the package "package" in tag "tag".
If type is specified, include the extended information for archives of that type.
"""
return self.session.listArchives(buildID=buildID, type=type)
def localCache(self, buildinfo, fileinfo, type=None):
"""
Access a file in the local cache. If the file does not exist, it's downloaded
from the server. Returns an open file object.
"""
local_pi = koji.PathInfo(self.buildreq_dir)
if type == 'win':
localpath = os.path.join(local_pi.winbuild(buildinfo),
fileinfo['relpath'], fileinfo['filename'])
else:
raise koji.BuildError, 'unsupported file type: %s' % type
if not os.path.isfile(localpath):
remote_pi = koji.PathInfo(self.options.topurl)
if type == 'win':
remote_url = remote_pi.winbuild(buildinfo) + '/' + \
fileinfo['relpath'] + '/' + fileinfo['filename']
else:
raise koji.BuildError, 'unsupported file type: %s' % type
koji.ensuredir(os.path.dirname(localpath))
urlgrabber.urlgrab(remote_url, filename=localpath)
return file(localpath, 'r')
def getFile(self, buildinfo, archiveinfo, offset, length, type=None):
"""
Get the contents of the file indicated by fileinfo, returning a maximum of
"length" bytes starting at "offset". Contents are returned base64-encoded.
"""
offset = int(offset)
length = int(length)
fileobj = self.localCache(buildinfo, archiveinfo, type=type)
try:
fileobj.seek(offset)
data = fileobj.read(length)
encoded = base64.b64encode(data)
del data
return encoded
finally:
fileobj.close()
def upload(self, path, offset, contents):
local_path = os.path.abspath(os.path.join(self.workdir, path))
if not local_path.startswith(self.workdir):
local_path = os.path.abspath(os.path.join(self.output_dir, path))
if not local_path.startswith(self.output_dir):
raise koji.BuildError, 'invalid upload path: %s' % path
koji.ensuredir(os.path.dirname(local_path))
# accept offset as a str to avoid problems with files larger than 2**32
@ -468,8 +560,8 @@ class VMExecTask(BaseTaskHandler):
return len(data)
def verifyChecksum(self, path, checksum, algo='sha1'):
local_path = os.path.abspath(os.path.join(self.workdir, path))
if not local_path.startswith(self.workdir):
local_path = os.path.abspath(os.path.join(self.output_dir, path))
if not local_path.startswith(self.output_dir):
raise koji.BuildError, 'invalid path: %s' % path
if not os.path.isfile(local_path):
raise koji.BuildError, '%s does not exist' % local_path
@ -578,7 +670,7 @@ class VMExecTask(BaseTaskHandler):
else:
vm.destroy()
self.server.server_close()
self.uploadTree(self.workdir)
self.uploadTree(self.output_dir)
if self.success:
return self.output
else:

3
vm/kojivmd.conf
View file

@ -14,6 +14,9 @@
; The directory root for temporary storage
; workdir=/tmp/koji
; The url where the Koji root directory (/mnt/koji) can be accessed
topurl=http://koji.example.com/kojiroot
; The URL for the xmlrpc server
server=http://hub.example.com/kojihub