particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of ssh://mikem@git.fedoraproject.org/git/hosted/koji

Browse source
This commit is contained in:
Mike McLean 2007-03-28 15:11:10 -04:00
commit d477abf7fd
6 changed files with 107 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

72
builder/kojid
View file

@ -212,32 +212,33 @@ def scan_mounts(topdir):
def incrementalUpload(fname, fd, path, retries=5, logger=None):
if not fd:
return
offset = fd.tell()
contents = fd.read()
size = len(contents)
if size == 0:
return
data = base64.encodestring(contents)
digest = md5.new(contents).hexdigest()
del contents
tries = 0
while True:
if session.uploadFile(path, fname, size, digest, offset, data):
return
if tries <= retries:
tries += 1
time.sleep(10)
continue
else:
if logger:
logger.error("Error uploading file %s to %s at offset %d" % (fname, path, offset))
offset = fd.tell()
contents = fd.read(65536)
size = len(contents)
if size == 0:
break
data = base64.encodestring(contents)
digest = md5.new(contents).hexdigest()
del contents
tries = 0
while True:
if session.uploadFile(path, fname, size, digest, offset, data):
break
if tries <= retries:
tries += 1
time.sleep(10)
continue
else:
sys.stderr.write("Error uploading file %s to %s at offset %d" % (fname, path, offset))
sys.stderr.write("\n")
return
if logger:
logger.error("Error uploading file %s to %s at offset %d" % (fname, path, offset))
else:
sys.stderr.write("Error uploading file %s to %s at offset %d\n" % (fname, path, offset))
break
def _parseStatus(rv, prefix):
if isinstance(prefix, list) or isinstance(prefix, tuple):
@ -352,8 +353,8 @@ class BuildRoot(object):
logs = {}
while True:
status = os.waitpid(pid,os.WNOHANG)
time.sleep(1)
status = os.waitpid(pid, os.WNOHANG)
try:
results = os.listdir(resultdir)
@ -363,19 +364,22 @@ class BuildRoot(object):
for fname in results:
if fname.endswith('.log') and not logs.has_key(fname):
logs[fname] = (None, None)
logs[fname] = (None, None, 0)
for (fname, (fd, inode)) in logs.items():
for (fname, (fd, inode, size)) in logs.items():
try:
fpath = os.path.join(resultdir, fname)
stat_info = os.stat(fpath)
if stat_info.st_ino != inode:
if not fd or stat_info.st_ino != inode or stat_info.st_size < size:
# either a file we haven't opened before, or mock replaced a file we had open with
# a new file and is writing to it, but our fd is pointing to the old file
# a new file and is writing to it, or truncated the file we're reading,
# but our fd is pointing to the previous location in the old file
if fd:
self.logger.info('Rereading %s, inode: %s -> %s, size: %s -> %s' %
(fpath, inode, stat_info.st_ino, size, stat_info.st_size))
fd.close()
fd = file(fpath, 'r')
logs[fname] = (fd, stat_info.st_ino)
logs[fname] = (fd, stat_info.st_ino, stat_info.st_size)
except:
self.logger.error("Error reading mock log: %s", fpath)
self.logger.error(''.join(traceback.format_exception(*sys.exc_info())))
@ -384,7 +388,7 @@ class BuildRoot(object):
incrementalUpload(fname, fd, uploadpath, self.logger)
if status[0] != 0:
for (fname, (fd, inode)) in logs.items():
for (fname, (fd, inode, size)) in logs.items():
if fd:
fd.close()
return status[1]
@ -2359,9 +2363,9 @@ def get_options():
'workdir': '/tmp/koji',
'mockdir': '/var/lib/mock',
'mockuser': 'kojibuilder',
'packager': 'koji',
'vendor': 'koji',
'mockhost': 'linux-gnu',
'packager': 'Koji',
'vendor': 'Koji',
'mockhost': 'koji-linux-gnu',
'smtphost': 'mail@example.com',
'from_addr': 'Koji Build System <buildsys@example.com>',
'krb_principal': None,

6
builder/kojid.conf
View file

@ -21,13 +21,13 @@
; mockuser=kojibuilder
; The vendor to use in rpm headers
; vendor=koji
; vendor=Koji
; The packager to use in rpm headers
; packager=koji
; packager=Koji
; The _host string to use in mock
; mockhost=
; mockhost=koji-linux-gnu
; The URL for the xmlrpc server

29
cli/koji
View file

@ -708,14 +708,25 @@ def handle_call(options, session, args):
name = args[0]
non_kw = []
kw = {}
for arg in args[1:]:
if arg.isdigit():
non_kw.append(int(arg))
elif arg.find('=') != -1:
key, value = arg.split('=', 1)
kw[key] = value
def _convarg(val):
valmap = {'None': None,
'True': True,
'False': False}
if val.isdigit():
return int(val)
elif valmap.has_key(val):
return valmap[val]
else:
non_kw.append(arg)
return val
for arg in args[1:]:
if arg.find('=') != -1:
key, value = arg.split('=', 1)
kw[key] = _convarg(value)
else:
non_kw.append(_convarg(arg))
pprint.pprint(getattr(session, name).__call__(*non_kw, **kw))
def anon_handle_mock_config(options, session, args):
@ -731,6 +742,8 @@ def anon_handle_mock_config(options, session, args):
help=_("Specify mockdir"))
parser.add_option("--topdir", metavar="DIR",
help=_("Specify topdir"))
parser.add_option("--topurl", metavar="URL",
help=_("url under which Koji files are accessible"))
parser.add_option("--distribution", default="Koji Testing",
help=_("Change the distribution macro"))
parser.add_option("-o", metavar="FILE", dest="ofile", help=_("Output to a file"))
@ -742,7 +755,7 @@ def anon_handle_mock_config(options, session, args):
name = args[0]
arch = None
opts = {}
for k in ('topdir', 'distribution', 'mockdir'):
for k in ('topdir', 'topurl', 'distribution', 'mockdir'):
if hasattr(options, k):
opts[k] = getattr(options, k)
if options.buildroot:

28
hub/kojixmlrpc.py
View file

@ -136,23 +136,29 @@ class ModXMLRPCRequestHandler(object):
#might be ok, depending on method
if method not in ('exclusiveSession','login', 'krbLogin', 'logout'):
raise
if context.opts.get('LockOut',False) in ('yes', 'on', 'true', '1') and \
if context.opts.get('LockOut', 'no').lower() in ('yes', 'on', 'true', '1') and \
method not in ('login', 'krbLogin', 'sslLogin', 'logout'):
if not context.session.hasPerm('admin'):
raise koji.GenericError, "Server disabled for maintenance"
# handle named parameters
params,opts = koji.decode_args(*params)
sys.stderr.write("Handling method %s for session %s (#%s)\n" \
% (method, context.session.id, context.session.callnum))
if method != 'uploadFile' and context.opts.get('KojiDebug',False) in ('yes', 'on', 'true', '1'):
sys.stderr.write("Params: %s\n" % pprint.pformat(params))
sys.stderr.write("Opts: %s\n" % pprint.pformat(opts))
start = time.time()
if context.opts.get('KojiDebug', 'no').lower() in ('yes', 'on', 'true', '1'):
sys.stderr.write("Handling method %s for session %s (#%s)\n" \
% (method, context.session.id, context.session.callnum))
if method != 'uploadFile':
sys.stderr.write("Params: %s\n" % pprint.pformat(params))
sys.stderr.write("Opts: %s\n" % pprint.pformat(opts))
start = time.time()
ret = func(*params,**opts)
sys.stderr.write("Completed method %s for session %s (#%s): %f seconds\n"
% (method, context.session.id, context.session.callnum,
time.time()-start))
sys.stderr.flush()
if context.opts.get('KojiDebug', 'no').lower() in ('yes', 'on', 'true', '1'):
sys.stderr.write("Completed method %s for session %s (#%s): %f seconds\n"
% (method, context.session.id, context.session.callnum,
time.time()-start))
sys.stderr.flush()
return ret
def multiCall(self, calls):

10
koji.spec
View file

@ -1,6 +1,6 @@
%{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")}
%define baserelease 2
%define baserelease 4
#build with --define 'testbuild 1' to have a timestamp appended to release
%if x%{?testbuild} == x1
%define release %{baserelease}.%(date +%%Y%%m%%d.%%H%%M.%%S)
@ -151,6 +151,14 @@ if [ $1 = 0 ]; then
fi
%changelog
* Wed Mar 28 2007 Mike Bonnet <mikeb@redhat.com> - 0.9.7-4
- set SSL connection timeout to 12 hours
* Wed Mar 28 2007 Mike Bonnet <mikeb@redhat.com> - 0.9.7-3
- avoid SSL renegotiation
- improve log file handling in kojid
- bug fixes in command-line and web UI
* Sun Mar 25 2007 Mike Bonnet <mikeb@redhat.com> - 0.9.7-2
- enable http access to packages in kojid
- add Requires: pyOpenSSL

34
koji/__init__.py
View file

@ -885,7 +885,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts)
repodir = pathinfo.repo(repoid,tag_name)
url = "%s/%s" % (repodir,arch)
else:
pathinfo = PathInfo(topdir=opts.get('topdir'))
pathinfo = PathInfo(topdir=opts.get('topdir', '/mnt/koji'))
repodir = pathinfo.repo(repoid,tag_name)
url = "file://%s/%s" % (repodir,arch)
if managed:
@ -933,14 +933,14 @@ baseurl=%(url)s
macros = {
'_topdir' : '%s/build' % config_opts['chroothome'],
'_rpmfilename' : '%%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm',
'vendor' : opts['vendor'],
'packager' : opts['packager'],
'vendor' : opts.get('vendor', 'Koji'),
'packager' : opts.get('packager', 'Koji'),
'_host_cpu' : arch,
'_host': '%s-%s' % (arch, opts['mockhost']),
'_host': '%s-%s' % (arch, opts.get('mockhost', 'koji-linux-gnu')),
#TODO - track some of these in the db instead?
}
if opts.has_key('distribution'):
macros['distribution'] = opts['distribution']
macros['distribution'] = opts.get('distribution')
config_opts['macros'] = '\n'.join(["%%%s %s" % (k, v) for k,v in macros.iteritems()])
parts = ["""#!/usr/bin/python -tt
@ -1081,13 +1081,15 @@ class ClientSession(object):
opts = opts.copy()
self.opts = opts
self.proxyOpts = {'allow_none':1}
if self.opts.get('debug_xmlrpc', False):
self.proxyOpts['verbose']=1
if self.opts.get('certs', False):
if self.opts.get('debug_xmlrpc'):
self.proxyOpts['verbose'] = 1
if self.opts.get('certs'):
self.proxyOpts['certs'] = self.opts['certs']
self.proxyClass = ssl.XMLRPCServerProxy.PlgXMLRPCServerProxy
else:
self.proxyClass = xmlrpclib.ServerProxy
if self.opts.get('timeout'):
self.proxyOpts['timeout'] = self.opts['timeout']
self.baseurl = baseurl
self.setSession(sinfo)
self.multicall = False
@ -1104,10 +1106,10 @@ class ClientSession(object):
# undo state changes made by ssl_login()
if self.baseurl.startswith('https:'):
self.baseurl = self.baseurl.replace('https:', 'http:')
if self.opts.get('certs', False):
del self.opts['certs']
if self.proxyOpts.get('certs', False):
del self.proxyOpts['certs']
self.opts.pop('certs', None)
self.proxyOpts.pop('certs', None)
self.opts.pop('timeout', None)
self.proxyOpts.pop('timeout', None)
self.proxyClass = xmlrpclib.ServerProxy
url = self.baseurl
else:
@ -1220,15 +1222,17 @@ class ClientSession(object):
certs['ca_cert'] = ca
certs['peer_ca_cert'] = serverca
# only use a timeout during login
# 60 second timeout during login
self.proxy = ssl.XMLRPCServerProxy.PlgXMLRPCServerProxy(self.baseurl, certs, timeout=60, **self.proxyOpts)
sinfo = self.callMethod('sslLogin', proxyuser)
if not sinfo:
raise AuthError, 'unable to obtain a session'
self.proxyClass = ssl.XMLRPCServerProxy.PlgXMLRPCServerProxy
self.opts['certs'] = certs
self.proxyOpts['certs'] = certs
self.opts['certs'] = self.proxyOpts['certs'] = certs
# 12 hour connection timeout. Some Koji operations can take a long time to return,
# but after 12 hours we can assume something is seriously wrong.
self.opts['timeout'] = self.proxyOpts['timeout'] = 60 * 60 * 12
self.setSession(sinfo)
return True