915 lines
33 KiB
Python
915 lines
33 KiB
Python
# kojixmlrpc - an XMLRPC interface for koji.
|
|
# Copyright (c) 2005-2014 Red Hat, Inc.
|
|
#
|
|
# Koji is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU Lesser General Public
|
|
# License as published by the Free Software Foundation;
|
|
# version 2.1 of the License.
|
|
#
|
|
# This software is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# Lesser General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Lesser General Public
|
|
# License along with this software; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
#
|
|
# Authors:
|
|
# Mike McLean <mikem@redhat.com>
|
|
|
|
import datetime
|
|
import inspect
|
|
import logging
|
|
import os
|
|
import pprint
|
|
import resource
|
|
import sys
|
|
import threading
|
|
import time
|
|
import traceback
|
|
import re
|
|
|
|
import koji
|
|
import koji.plugin
|
|
import koji.policy
|
|
import koji.util
|
|
import kojihub
|
|
from koji.context import context
|
|
# import xmlrpclib functions from koji to use tweaked Marshaller
|
|
from koji.server import ServerError, BadRequest, RequestTimeout
|
|
from koji.xmlrpcplus import ExtendedMarshaller, Fault, dumps, getparser
|
|
from . import auth
|
|
from . import db
|
|
from . import scheduler
|
|
from . import repos
|
|
|
|
|
|
# HTTP headers included in every request
|
|
GLOBAL_HEADERS = [
|
|
('Koji-Version', koji.__version__),
|
|
]
|
|
|
|
|
|
class Marshaller(ExtendedMarshaller):
|
|
|
|
dispatch = ExtendedMarshaller.dispatch.copy()
|
|
|
|
def dump_datetime(self, value, write):
|
|
# For backwards compatibility, we return datetime objects as strings
|
|
value = value.isoformat(' ')
|
|
self.dump_unicode(value, write)
|
|
dispatch[datetime.datetime] = dump_datetime
|
|
|
|
|
|
class HandlerRegistry(object):
|
|
"""Track handlers for RPC calls"""
|
|
|
|
def __init__(self):
|
|
self.funcs = {}
|
|
# introspection functions
|
|
self.register_function(self.list_api, name="_listapi")
|
|
self.register_function(self.system_listMethods, name="system.listMethods")
|
|
self.register_function(self.system_methodSignature, name="system.methodSignature")
|
|
self.register_function(self.system_methodHelp, name="system.methodHelp")
|
|
self.argspec_cache = {}
|
|
|
|
def register_function(self, function, name=None):
|
|
if name is None:
|
|
name = function.__name__
|
|
self.funcs[name] = function
|
|
|
|
def register_module(self, instance, prefix=None):
|
|
"""Register all the public functions in an instance with prefix prepended
|
|
|
|
For example
|
|
h.register_module(exports,"pub.sys")
|
|
will register the methods of exports with names like
|
|
pub.sys.method1
|
|
pub.sys.method2
|
|
...etc
|
|
"""
|
|
for name in dir(instance):
|
|
if name.startswith('_'):
|
|
continue
|
|
function = getattr(instance, name)
|
|
if not callable(function):
|
|
continue
|
|
if prefix is not None:
|
|
name = "%s.%s" % (prefix, name)
|
|
self.register_function(function, name=name)
|
|
|
|
def register_instance(self, instance):
|
|
self.register_module(instance)
|
|
|
|
def register_plugin(self, plugin):
|
|
"""Scan a given plugin for handlers
|
|
|
|
Handlers are functions marked with one of the decorators defined in koji.plugin
|
|
"""
|
|
for v in vars(plugin).values():
|
|
if isinstance(v, type):
|
|
# skip classes
|
|
continue
|
|
if callable(v):
|
|
if getattr(v, 'exported', False):
|
|
if hasattr(v, 'export_alias'):
|
|
name = getattr(v, 'export_alias')
|
|
else:
|
|
name = v.__name__
|
|
self.register_function(v, name=name)
|
|
if getattr(v, 'callbacks', None):
|
|
for cbtype in v.callbacks:
|
|
koji.plugin.register_callback(cbtype, v)
|
|
|
|
def getargspec(self, func):
|
|
ret = self.argspec_cache.get(func)
|
|
if ret:
|
|
return ret
|
|
ret = tuple(inspect.getfullargspec(func))
|
|
if inspect.ismethod(func) and func.__self__:
|
|
# bound method, remove first arg
|
|
args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = ret
|
|
if args:
|
|
aname = args[0] # generally "self"
|
|
del args[0]
|
|
if defaults and aname in defaults:
|
|
# shouldn't happen, but...
|
|
del defaults[aname]
|
|
self.argspec_cache[func] = ret
|
|
return ret
|
|
|
|
def list_api(self):
|
|
"""List available API calls"""
|
|
funcs = []
|
|
for name, func in self.funcs.items():
|
|
# the keys in self.funcs determine the name of the method as seen over xmlrpc
|
|
# func.__name__ might differ (e.g. for dotted method names)
|
|
sig = inspect.signature(func)
|
|
args = []
|
|
argdesc = []
|
|
for pname, param in sig.parameters.items():
|
|
if param.default != param.empty:
|
|
args.append([pname, param.default])
|
|
else:
|
|
args.append(pname)
|
|
argdesc = '(%s)' % ', '.join([str(x) for x in sig.parameters.values()])
|
|
argspec = self.getargspec(func)
|
|
funcs.append({'name': name,
|
|
'doc': func.__doc__,
|
|
'argspec': argspec,
|
|
'argdesc': argdesc,
|
|
'args': args})
|
|
return funcs
|
|
|
|
def system_listMethods(self):
|
|
return list(self.funcs.keys())
|
|
|
|
def system_methodSignature(self, method):
|
|
# it is not possible to autogenerate this data
|
|
return 'signatures not supported'
|
|
|
|
def system_methodHelp(self, method):
|
|
func = self.funcs.get(method)
|
|
if func is None:
|
|
return ""
|
|
sig = inspect.signature(func)
|
|
args = ', '.join([str(x) for x in sig.parameters.values()])
|
|
ret = f'{method}({args})'
|
|
if func.__doc__:
|
|
ret += f'\ndescription: {func.__doc__}'
|
|
return ret
|
|
|
|
def get(self, name):
|
|
func = self.funcs.get(name, None)
|
|
if func is None:
|
|
raise koji.GenericError("Invalid method: %s" % name)
|
|
return func
|
|
|
|
|
|
class HandlerAccess(object):
|
|
"""This class is used to grant access to the rpc handlers"""
|
|
|
|
def __init__(self, registry):
|
|
self.__reg = registry
|
|
|
|
def call(self, __name, *args, **kwargs):
|
|
return self.__reg.get(__name)(*args, **kwargs)
|
|
|
|
def get(self, name):
|
|
return self.__reg.get(name)
|
|
|
|
|
|
class ModXMLRPCRequestHandler(object):
|
|
"""Simple XML-RPC handler for mod_wsgi environment"""
|
|
|
|
def __init__(self, handlers):
|
|
self.traceback = False
|
|
self.handlers = handlers # expecting HandlerRegistry instance
|
|
self.logger = logging.getLogger('koji.xmlrpc')
|
|
|
|
def _get_handler(self, name):
|
|
# just a wrapper so we can handle multicall ourselves
|
|
# we don't register multicall since the registry will outlive our instance
|
|
if name in ('multiCall', 'system.multicall'):
|
|
return self.multiCall
|
|
else:
|
|
return self.handlers.get(name)
|
|
|
|
def _read_request(self, stream):
|
|
parser, unmarshaller = getparser()
|
|
rlen = 0
|
|
maxlen = opts.get('MaxRequestLength', None)
|
|
while True:
|
|
try:
|
|
chunk = stream.read(8192)
|
|
except OSError as e:
|
|
str_e = str(e)
|
|
if 'timeout' in str_e:
|
|
self.logger.exception("Timed out reading input stream. Content-Length: "
|
|
f"{context.environ.get('CONTENT_LENGTH')}")
|
|
raise RequestTimeout(str_e)
|
|
else:
|
|
self.logger.exception("Error reading input stream. Content-Length: "
|
|
f"{context.environ.get('CONTENT_LENGTH')}")
|
|
raise BadRequest(str_e)
|
|
if not chunk:
|
|
break
|
|
rlen += len(chunk)
|
|
if maxlen and rlen > maxlen:
|
|
raise koji.GenericError('Request too long')
|
|
parser.feed(chunk)
|
|
if rlen == 0:
|
|
# blank posts are unfortunately a common client mistake
|
|
# make them less noisy
|
|
self.logger.debug('Empty request')
|
|
raise BadRequest('Empty request')
|
|
parser.close()
|
|
return unmarshaller.close(), unmarshaller.getmethodname()
|
|
|
|
def _log_exception(self):
|
|
e_class, e = sys.exc_info()[:2]
|
|
faultCode = getattr(e_class, 'faultCode', 1)
|
|
tb_type = context.opts.get('KojiTraceback', None)
|
|
tb_str = ''.join(traceback.format_exception(*sys.exc_info()))
|
|
if issubclass(e_class, koji.GenericError):
|
|
if context.opts.get('KojiDebug'):
|
|
if tb_type == "extended":
|
|
faultString = koji.format_exc_plus()
|
|
else:
|
|
faultString = tb_str
|
|
else:
|
|
faultString = str(e)
|
|
else:
|
|
if tb_type == "normal":
|
|
faultString = tb_str
|
|
elif tb_type == "extended":
|
|
faultString = koji.format_exc_plus()
|
|
else:
|
|
faultString = "%s: %s" % (e_class, e)
|
|
self.logger.warning(tb_str)
|
|
return faultCode, faultString
|
|
|
|
def _wrap_handler(self, handler, environ):
|
|
"""Catch exceptions and encode response of handler"""
|
|
|
|
# generate response
|
|
try:
|
|
response = handler(environ)
|
|
# wrap response in a singleton tuple
|
|
response = (response,)
|
|
response = dumps(response, methodresponse=1, marshaller=Marshaller)
|
|
except ServerError:
|
|
raise
|
|
# these are handled higher up
|
|
except Fault as fault:
|
|
self.traceback = True
|
|
response = dumps(fault, marshaller=Marshaller)
|
|
except Exception:
|
|
self.traceback = True
|
|
# report exception back to server
|
|
faultCode, faultString = self._log_exception()
|
|
response = dumps(Fault(faultCode, faultString), marshaller=Marshaller)
|
|
|
|
return response
|
|
|
|
def handle_upload(self, environ):
|
|
# uploads can't be in a multicall
|
|
context.method = None
|
|
self.check_session()
|
|
self.enforce_lockout()
|
|
return kojihub.handle_upload(environ)
|
|
|
|
def handle_rpc(self, environ):
|
|
try:
|
|
params, method = self._read_request(environ['wsgi.input'])
|
|
except Exception as e:
|
|
# only log full trace in debug stream
|
|
self.logger.debug("Error reading request", exc_info=True)
|
|
raise BadRequest('Invalid request: %s' % e)
|
|
return self._dispatch(method, params)
|
|
|
|
def check_session(self):
|
|
if not hasattr(context, "session"):
|
|
# we may be called again by one of our meta-calls (like multiCall)
|
|
# so we should only create a session if one does not already exist
|
|
context.session = auth.Session()
|
|
try:
|
|
context.session.validate()
|
|
except koji.AuthLockError:
|
|
# might be ok, depending on method
|
|
if context.method not in ('exclusiveSession', 'login', 'logout'):
|
|
raise
|
|
|
|
def enforce_lockout(self):
|
|
if context.opts.get('LockOut') and \
|
|
context.method not in ('login', 'sslLogin', 'logout') and \
|
|
not context.session.hasPerm('admin'):
|
|
raise koji.ServerOffline("Server disabled for maintenance")
|
|
|
|
def _dispatch(self, method, params):
|
|
func = self._get_handler(method)
|
|
context.method = method
|
|
context.params = params
|
|
self.check_session()
|
|
self.enforce_lockout()
|
|
# handle named parameters
|
|
params, opts = koji.decode_args(*params)
|
|
|
|
if self.logger.isEnabledFor(logging.INFO):
|
|
self.logger.info("Handling method %s for session %s (#%s)",
|
|
method, context.session.id, context.session.callnum)
|
|
if method != 'uploadFile' and self.logger.isEnabledFor(logging.DEBUG):
|
|
self.logger.debug("Params: %s", pprint.pformat(params))
|
|
self.logger.debug("Opts: %s", pprint.pformat(opts))
|
|
start = time.time()
|
|
|
|
ret = koji.util.call_with_argcheck(func, params, opts)
|
|
|
|
if self.logger.isEnabledFor(logging.INFO):
|
|
rusage = resource.getrusage(resource.RUSAGE_SELF)
|
|
self.logger.info(
|
|
"Completed method %s for session %s (#%s): %f seconds, rss %s, stime %f",
|
|
method, context.session.id, context.session.callnum,
|
|
time.time() - start,
|
|
rusage.ru_maxrss, rusage.ru_stime)
|
|
|
|
return ret
|
|
|
|
def multiCall(self, calls):
|
|
"""Execute a multicall. Execute each method call in the calls list, collecting
|
|
results and errors, and return those as a list."""
|
|
results = []
|
|
for call in calls:
|
|
savepoint = db.Savepoint('multiCall_loop')
|
|
try:
|
|
result = self._dispatch(call['methodName'], call['params'])
|
|
except Fault as fault:
|
|
savepoint.rollback()
|
|
results.append({'faultCode': fault.faultCode, 'faultString': fault.faultString})
|
|
except Exception:
|
|
savepoint.rollback()
|
|
# transform unknown exceptions into XML-RPC Faults
|
|
# don't create a reference to full traceback since this creates
|
|
# a circular reference.
|
|
exc_type, exc_value = sys.exc_info()[:2]
|
|
faultCode = getattr(exc_type, 'faultCode', 1)
|
|
faultString = ', '.join([str(arg) for arg in exc_value.args])
|
|
trace = traceback.format_exception(*sys.exc_info())
|
|
# traceback is not part of the multicall spec,
|
|
# but we include it for debugging purposes
|
|
results.append({'faultCode': faultCode,
|
|
'faultString': faultString,
|
|
'traceback': trace})
|
|
self._log_exception()
|
|
else:
|
|
results.append([result])
|
|
|
|
# subsequent calls should not recycle event ids
|
|
if hasattr(context, 'event_id'):
|
|
del context.event_id
|
|
|
|
return results
|
|
|
|
def handle_request(self, req):
|
|
"""Handle a single XML-RPC request"""
|
|
|
|
pass
|
|
# XXX no longer used
|
|
|
|
|
|
def offline_reply(start_response, msg=None):
|
|
"""Send a ServerOffline reply"""
|
|
faultCode = koji.ServerOffline.faultCode
|
|
if msg is None:
|
|
faultString = "server is offline"
|
|
else:
|
|
faultString = msg
|
|
response = dumps(Fault(faultCode, faultString)).encode()
|
|
headers = GLOBAL_HEADERS + [
|
|
('Content-Length', str(len(response))),
|
|
('Content-Type', "text/xml"),
|
|
]
|
|
start_response('200 OK', headers)
|
|
return [response]
|
|
|
|
|
|
def error_reply(start_response, status, response, extra_headers=None):
|
|
response = response.encode()
|
|
headers = GLOBAL_HEADERS + [
|
|
('Content-Length', str(len(response))),
|
|
('Content-Type', "text/plain"),
|
|
]
|
|
if extra_headers:
|
|
headers.extend(extra_headers)
|
|
start_response(status, headers)
|
|
return [response]
|
|
|
|
|
|
# Known configuration options, used by load_config
|
|
config_map = [
|
|
# option, type, default
|
|
['DBName', 'string', None],
|
|
['DBUser', 'string', None],
|
|
['DBHost', 'string', None],
|
|
['DBhost', 'string', None], # alias for backwards compatibility
|
|
['DBPort', 'integer', None],
|
|
['DBPass', 'string', None],
|
|
['DBConnectionString', 'string', None],
|
|
['KojiDir', 'string', None],
|
|
|
|
['ProxyPrincipals', 'string', ''],
|
|
['HostPrincipalFormat', 'string', None],
|
|
['AllowedKrbRealms', 'string', '*'],
|
|
# TODO: this option should be turned True in 1.34
|
|
['DisableURLSessions', 'boolean', False],
|
|
|
|
['DNUsernameComponent', 'string', 'CN'],
|
|
['ProxyDNs', 'string', ''],
|
|
|
|
['CheckClientIP', 'boolean', True],
|
|
|
|
['LoginCreatesUser', 'boolean', True],
|
|
['AllowProxyAuthType', 'boolean', False],
|
|
['KojiWebURL', 'string', 'http://localhost.localdomain/koji'],
|
|
['EmailDomain', 'string', None],
|
|
['NotifyOnSuccess', 'boolean', True],
|
|
['DisableNotifications', 'boolean', False],
|
|
|
|
['Plugins', 'string', ''],
|
|
['PluginPath', 'string', '/usr/lib/koji-hub-plugins'],
|
|
|
|
['KojiDebug', 'boolean', False],
|
|
['KojiTraceback', 'string', None],
|
|
['VerbosePolicy', 'boolean', False],
|
|
|
|
['LogLevel', 'string', 'WARNING'],
|
|
['LogFormat', 'string',
|
|
'%(asctime)s [%(levelname)s] m=%(method)s u=%(user_name)s p=%(process)s r=%(remoteaddr)s '
|
|
'%(name)s: %(message)s'],
|
|
|
|
['MissingPolicyOk', 'boolean', True],
|
|
['EnableMaven', 'boolean', False],
|
|
['EnableWin', 'boolean', False],
|
|
|
|
['RLIMIT_AS', 'string', None],
|
|
['RLIMIT_CORE', 'string', None],
|
|
['RLIMIT_CPU', 'string', None],
|
|
['RLIMIT_DATA', 'string', None],
|
|
['RLIMIT_FSIZE', 'string', None],
|
|
['RLIMIT_MEMLOCK', 'string', None],
|
|
['RLIMIT_NOFILE', 'string', None],
|
|
['RLIMIT_NPROC', 'string', None],
|
|
['RLIMIT_OFILE', 'string', None], # alias for RLIMIT_NOFILE
|
|
['RLIMIT_RSS', 'string', None],
|
|
['RLIMIT_STACK', 'string', None],
|
|
|
|
['MemoryWarnThreshold', 'integer', 5000],
|
|
['MaxRequestLength', 'integer', 4194304],
|
|
|
|
['LockOut', 'boolean', False],
|
|
['ServerOffline', 'boolean', False],
|
|
['OfflineMessage', 'string', None],
|
|
|
|
['MaxNameLengthInternal', 'integer', 256],
|
|
['RegexNameInternal', 'string', r'^[A-Za-z0-9/_.+-]+$'],
|
|
['RegexUserName', 'string', r'^[A-Za-z0-9/_.@-]+$'],
|
|
|
|
['RPMDefaultChecksums', 'string', 'md5 sha256'],
|
|
|
|
['SessionRenewalTimeout', 'integer', 1440],
|
|
|
|
# scheduler options
|
|
['MaxJobs', 'integer', 15],
|
|
['CapacityOvercommit', 'integer', 5],
|
|
['ReadyTimeout', 'integer', 180],
|
|
['AssignTimeout', 'integer', 300],
|
|
['SoftRefusalTimeout', 'integer', 900],
|
|
['HostTimeout', 'integer', 900],
|
|
['RunInterval', 'integer', 60],
|
|
|
|
# repo options
|
|
['MaxRepoTasks', 'integer', 10],
|
|
['MaxRepoTasksMaven', 'integer', 2],
|
|
['RepoRetries', 'integer', 3],
|
|
['RequestCleanTime', 'integer', 60 * 24], # in minutes
|
|
['AllowNewRepo', 'boolean', True],
|
|
['RepoLag', 'integer', 3600],
|
|
['RepoAutoLag', 'integer', 7200],
|
|
['RepoLagWindow', 'integer', 600],
|
|
['RepoQueueUser', 'string', 'kojira'],
|
|
['DebuginfoTags', 'string', ''],
|
|
['SourceTags', 'string', ''],
|
|
['SeparateSourceTags', 'string', ''],
|
|
]
|
|
|
|
|
|
def load_config(environ):
|
|
"""Load configuration options
|
|
|
|
Options are read from a config file. The config file location is
|
|
controlled by the koji.hub.ConfigFile environment variable
|
|
in the httpd config. To override this (for example):
|
|
|
|
SetEnv koji.hub.ConfigFile /home/developer/koji/hub/hub.conf
|
|
|
|
Backwards compatibility:
|
|
- if ConfigFile is not set, opts are loaded from http config
|
|
- if ConfigFile is set, then the http config must not provide Koji options
|
|
- In a future version we will load the default hub config regardless
|
|
- all PythonOptions (except ConfigFile) are now deprecated and support for them
|
|
will disappear in a future version of Koji
|
|
"""
|
|
# get our config file(s)
|
|
cf = environ.get('koji.hub.ConfigFile', '/etc/koji-hub/hub.conf')
|
|
cfdir = environ.get('koji.hub.ConfigDir', '/etc/koji-hub/hub.conf.d')
|
|
config = koji.read_config_files([cfdir, (cf, True)], raw=True)
|
|
|
|
opts = {}
|
|
for name, dtype, default in config_map:
|
|
key = ('hub', name)
|
|
if config and config.has_option(*key):
|
|
if dtype == 'integer':
|
|
opts[name] = config.getint(*key)
|
|
elif dtype == 'boolean':
|
|
opts[name] = config.getboolean(*key)
|
|
elif dtype == 'string':
|
|
opts[name] = config.get(*key)
|
|
else:
|
|
# anything else is an error in the map definition
|
|
raise ValueError(f'Invalid data type {dtype} for {name} option')
|
|
continue
|
|
opts[name] = default
|
|
if opts['DBHost'] is None:
|
|
opts['DBHost'] = opts['DBhost']
|
|
if opts['RLIMIT_NOFILE'] is None:
|
|
opts['RLIMIT_NOFILE'] = opts['RLIMIT_OFILE']
|
|
# load policies
|
|
# (only from config file)
|
|
if config and config.has_section('policy'):
|
|
# for the moment, we simply transfer the policy conf to opts
|
|
opts['policy'] = dict(config.items('policy'))
|
|
else:
|
|
opts['policy'] = {}
|
|
for pname, text in _default_policies.items():
|
|
opts['policy'].setdefault(pname, text)
|
|
# use configured KojiDir
|
|
if opts.get('KojiDir') is not None:
|
|
koji.BASEDIR = opts['KojiDir']
|
|
koji.pathinfo.topdir = opts['KojiDir']
|
|
if opts['RegexNameInternal'] != '':
|
|
opts['RegexNameInternal.compiled'] = re.compile(opts['RegexNameInternal'])
|
|
if opts['RegexUserName'] != '':
|
|
opts['RegexUserName.compiled'] = re.compile(opts['RegexUserName'])
|
|
|
|
return opts
|
|
|
|
|
|
def load_plugins(opts):
|
|
"""Load plugins specified by our configuration"""
|
|
if not opts['Plugins']:
|
|
return
|
|
logger = logging.getLogger('koji.plugins')
|
|
tracker = koji.plugin.PluginTracker(path=opts['PluginPath'].split(':'))
|
|
for name in opts['Plugins'].split():
|
|
logger.info('Loading plugin: %s', name)
|
|
try:
|
|
tracker.load(name)
|
|
except Exception:
|
|
logger.error(''.join(traceback.format_exception(*sys.exc_info())))
|
|
# make this non-fatal, but set ServerOffline
|
|
opts['ServerOffline'] = True
|
|
opts['OfflineMessage'] = 'configuration error'
|
|
return tracker
|
|
|
|
|
|
_default_policies = {
|
|
'build_rpm': '''
|
|
all :: allow
|
|
''',
|
|
'build_from_srpm': '''
|
|
has_perm admin :: allow
|
|
all :: deny Only admin can do this via default policy
|
|
''',
|
|
'build_from_repo_id': '''
|
|
has_perm admin :: allow
|
|
all :: deny Only admin can do this via default policy
|
|
''',
|
|
'build_from_scm': '''
|
|
has_perm admin :: allow
|
|
# match scm_type CVS CVS+SSH && match scm_host scm.example.com && match scm_repository /cvs/example :: allow
|
|
# match scm_type GIT GIT+SSH && match scm_host git.example.org && match scm_repository /example :: allow
|
|
# match scm_type SVN SVN+SSH && match scm_host svn.example.org && match scm_repository /users/* :: allow
|
|
all :: deny Only admin can do this via default policy
|
|
''', # noqa: E501
|
|
'package_list': '''
|
|
has_perm admin :: allow
|
|
has_perm tag :: allow
|
|
all :: deny Only admin/tag can do this via default policy
|
|
''',
|
|
'channel': '''
|
|
has req_channel :: req
|
|
is_child_task :: parent
|
|
all :: default
|
|
''',
|
|
'vm': '''
|
|
has_perm admin win-admin :: allow
|
|
all :: deny Only admin/win-admin can do this via default policy
|
|
''',
|
|
'cg_import': '''
|
|
all :: allow
|
|
''',
|
|
'volume': '''
|
|
all :: DEFAULT
|
|
''',
|
|
'priority': '''
|
|
all :: stay
|
|
''',
|
|
'draft_promotion': '''
|
|
has_perm draft-promoter :: allow
|
|
is_build_owner :: allow
|
|
all :: deny Only draft-promoter and build owner can do this via default policy
|
|
'''
|
|
}
|
|
|
|
|
|
def get_policy(opts, plugins):
|
|
if not opts.get('policy'):
|
|
return
|
|
# first find available policy tests
|
|
alltests = [koji.policy.findSimpleTests([vars(kojihub), vars(koji.policy)])]
|
|
# we delay merging these to allow a test to be overridden for a specific policy
|
|
for plugin_name in opts.get('Plugins', '').split():
|
|
plugin = plugins.get(plugin_name)
|
|
if not plugin:
|
|
continue
|
|
alltests.append(koji.policy.findSimpleTests(vars(plugin)))
|
|
policy = {}
|
|
for pname, text in opts['policy'].items():
|
|
# filter/merge tests
|
|
merged = {}
|
|
for tests in alltests:
|
|
# tests can be limited to certain policies by setting a class variable
|
|
for name, test in tests.items():
|
|
if hasattr(test, 'policy'):
|
|
if isinstance(test.policy, str):
|
|
if pname != test.policy:
|
|
continue
|
|
elif pname not in test.policy:
|
|
continue
|
|
# in case of name overlap, last one wins
|
|
# hence plugins can override builtin tests
|
|
merged[name] = test
|
|
policy[pname] = koji.policy.SimpleRuleSet(text.splitlines(), merged)
|
|
return policy
|
|
|
|
|
|
class HubFormatter(logging.Formatter):
|
|
"""Support some koji specific fields in the format string"""
|
|
|
|
def format(self, record):
|
|
record.method = getattr(context, 'method', None)
|
|
if hasattr(context, 'environ'):
|
|
record.remoteaddr = "%s:%s" % (
|
|
context.environ.get('REMOTE_ADDR', '?'),
|
|
context.environ.get('REMOTE_PORT', '?'))
|
|
else:
|
|
record.remoteaddr = "?:?"
|
|
if hasattr(context, 'session'):
|
|
record.user_id = context.session.user_id
|
|
record.session_id = context.session.id
|
|
record.callnum = context.session.callnum
|
|
record.user_name = context.session.user_data.get('name')
|
|
else:
|
|
record.user_id = None
|
|
record.session_id = None
|
|
record.callnum = None
|
|
record.user_name = None
|
|
return logging.Formatter.format(self, record)
|
|
|
|
|
|
def setup_logging1():
|
|
"""Set up basic logging, before options are loaded"""
|
|
global log_handler
|
|
logger = logging.getLogger("koji")
|
|
logger.setLevel(logging.WARNING)
|
|
# stderr logging (stderr goes to httpd logs)
|
|
log_handler = logging.StreamHandler()
|
|
log_format = '%(asctime)s [%(levelname)s] SETUP p=%(process)s %(name)s: %(message)s'
|
|
log_handler.setFormatter(HubFormatter(log_format))
|
|
log_handler.setLevel(logging.DEBUG)
|
|
logger.addHandler(log_handler)
|
|
|
|
|
|
def setup_logging2(opts):
|
|
global log_handler
|
|
"""Adjust logging based on configuration options"""
|
|
# determine log level
|
|
level = opts['LogLevel']
|
|
valid_levels = ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')
|
|
# the config value can be a single level name or a series of
|
|
# logger:level names pairs. processed in order found
|
|
default = None
|
|
for part in level.split():
|
|
pair = part.split(':', 1)
|
|
if len(pair) == 2:
|
|
name, level = pair
|
|
else:
|
|
name = 'koji'
|
|
level = part
|
|
default = level
|
|
if level not in valid_levels:
|
|
raise koji.GenericError("Invalid log level: %s" % level)
|
|
# all our loggers start with koji
|
|
if name == '':
|
|
name = 'koji'
|
|
default = level
|
|
elif name.startswith('.'):
|
|
name = 'koji' + name
|
|
elif not name.startswith('koji'):
|
|
name = 'koji.' + name
|
|
level_code = logging.getLevelName(level)
|
|
logging.getLogger(name).setLevel(level_code)
|
|
logger = logging.getLogger("koji")
|
|
# if KojiDebug is set, force main log level to DEBUG
|
|
if opts.get('KojiDebug'):
|
|
logger.setLevel(logging.DEBUG)
|
|
elif default is None:
|
|
# LogLevel did not configure a default level
|
|
logger.setLevel(logging.WARNING)
|
|
# log_handler defined in setup_logging1
|
|
log_handler.setFormatter(HubFormatter(opts['LogFormat']))
|
|
|
|
|
|
def get_memory_usage():
|
|
pagesize = resource.getpagesize()
|
|
statm = [pagesize * int(y) // 1024
|
|
for y in "".join(open("/proc/self/statm").readlines()).strip().split()]
|
|
size, res, shr, text, lib, data, dirty = statm
|
|
return res - shr
|
|
|
|
|
|
def server_setup(environ):
|
|
global opts, plugins, registry, policy
|
|
logger = logging.getLogger('koji')
|
|
try:
|
|
setup_logging1()
|
|
opts = load_config(environ)
|
|
setup_logging2(opts)
|
|
koji.util.setup_rlimits(opts)
|
|
plugins = load_plugins(opts)
|
|
registry = get_registry(opts, plugins)
|
|
policy = get_policy(opts, plugins)
|
|
if opts.get('DBConnectionString'):
|
|
db.provideDBopts(dsn=opts['DBConnectionString'])
|
|
else:
|
|
db.provideDBopts(database=opts["DBName"],
|
|
user=opts["DBUser"],
|
|
password=opts.get("DBPass", None),
|
|
host=opts.get("DBHost", None),
|
|
port=opts.get("DBPort", None))
|
|
except Exception:
|
|
tb_str = ''.join(traceback.format_exception(*sys.exc_info()))
|
|
logger.error(tb_str)
|
|
opts = {
|
|
'ServerOffline': True,
|
|
'OfflineMessage': 'server startup error',
|
|
}
|
|
|
|
|
|
#
|
|
# wsgi handler
|
|
#
|
|
|
|
firstcall = True
|
|
firstcall_lock = threading.Lock()
|
|
|
|
|
|
def application(environ, start_response):
|
|
global firstcall
|
|
if firstcall:
|
|
with firstcall_lock:
|
|
# check again, another thread may be ahead of us
|
|
if firstcall:
|
|
server_setup(environ)
|
|
firstcall = False
|
|
# XMLRPC uses POST only. Reject anything else
|
|
if environ['REQUEST_METHOD'] != 'POST':
|
|
extra_headers = [
|
|
('Allow', 'POST'),
|
|
]
|
|
response = "Method Not Allowed\n" \
|
|
"This is an XML-RPC server. Only POST requests are accepted.\n"
|
|
return error_reply(start_response, '405 Method Not Allowed', response, extra_headers)
|
|
if opts.get('ServerOffline'):
|
|
return offline_reply(start_response, msg=opts.get("OfflineMessage", None))
|
|
# XXX check request length
|
|
# XXX most of this should be moved elsewhere
|
|
if 1:
|
|
try:
|
|
start = time.time()
|
|
memory_usage_at_start = get_memory_usage()
|
|
|
|
context._threadclear()
|
|
context.commit_pending = False
|
|
context.opts = opts
|
|
context.handlers = HandlerAccess(registry)
|
|
context.environ = environ
|
|
context.policy = policy
|
|
try:
|
|
context.cnx = db.connect()
|
|
except Exception:
|
|
return offline_reply(start_response, msg="database outage")
|
|
h = ModXMLRPCRequestHandler(registry)
|
|
try:
|
|
if environ.get('CONTENT_TYPE') == 'application/octet-stream':
|
|
response = h._wrap_handler(h.handle_upload, environ)
|
|
else:
|
|
response = h._wrap_handler(h.handle_rpc, environ)
|
|
except BadRequest as e:
|
|
return error_reply(start_response, '400 Bad Request', str(e) + '\n')
|
|
except RequestTimeout as e:
|
|
return error_reply(start_response, '408 Request Timeout', str(e) + '\n')
|
|
response = response.encode()
|
|
headers = GLOBAL_HEADERS + [
|
|
('Content-Length', str(len(response))),
|
|
('Content-Type', "text/xml"),
|
|
]
|
|
start_response('200 OK', headers)
|
|
if h.traceback:
|
|
# rollback
|
|
context.cnx.rollback()
|
|
elif context.commit_pending:
|
|
# Currently there is not much data we can provide to the
|
|
# pre/postCommit callbacks. The handler can access context at
|
|
# least
|
|
koji.plugin.run_callbacks('preCommit')
|
|
context.cnx.commit()
|
|
koji.plugin.run_callbacks('postCommit')
|
|
memory_usage_at_end = get_memory_usage()
|
|
if memory_usage_at_end - memory_usage_at_start > opts['MemoryWarnThreshold']:
|
|
paramstr = repr(getattr(context, 'params', 'UNKNOWN'))
|
|
if len(paramstr) > 120:
|
|
paramstr = paramstr[:117] + "..."
|
|
h.logger.warning(
|
|
"Memory usage of process %d grew from %d KiB to %d KiB (+%d KiB) processing "
|
|
"request %s with args %s" %
|
|
(os.getpid(), memory_usage_at_start, memory_usage_at_end,
|
|
memory_usage_at_end - memory_usage_at_start, context.method, paramstr))
|
|
h.logger.debug("Returning %d bytes after %f seconds", len(response),
|
|
time.time() - start)
|
|
finally:
|
|
# make sure context gets cleaned up
|
|
if hasattr(context, 'cnx'):
|
|
try:
|
|
context.cnx.close()
|
|
except Exception:
|
|
pass
|
|
context._threadclear()
|
|
return [response] # XXX
|
|
|
|
|
|
def get_registry(opts, plugins):
|
|
# Create and populate handler registry
|
|
registry = HandlerRegistry()
|
|
functions = kojihub.RootExports()
|
|
hostFunctions = kojihub.HostExports()
|
|
schedulerFunctions = scheduler.SchedulerExports()
|
|
repoFunctions = repos.RepoExports()
|
|
registry.register_instance(functions)
|
|
registry.register_module(hostFunctions, "host")
|
|
registry.register_module(schedulerFunctions, "scheduler")
|
|
registry.register_module(repoFunctions, "repo")
|
|
registry.register_function(auth.login)
|
|
registry.register_function(auth.sslLogin)
|
|
registry.register_function(auth.logout)
|
|
registry.register_function(auth.subsession)
|
|
registry.register_function(auth.logoutChild)
|
|
registry.register_function(auth.exclusiveSession)
|
|
registry.register_function(auth.sharedSession)
|
|
for name in opts.get('Plugins', '').split():
|
|
plugin = plugins.get(name)
|
|
if not plugin:
|
|
continue
|
|
registry.register_plugin(plugin)
|
|
return registry
|