basic plugin support for hub and builders

This commit is contained in:
Mike McLean 2008-08-20 18:37:53 -04:00
parent 0b8f313039
commit 389aa5ff5c
7 changed files with 452 additions and 5 deletions

View file

@ -1,4 +1,5 @@
SUBDIRS = lib
BINFILES = kojid
PYFILES = $(wildcard *.py)
@ -7,6 +8,8 @@ _default:
clean:
rm -f *.o *.so *.pyc *~
for d in $(SUBDIRS); do make -s -C $$d clean; done
install:
@if [ "$(DESTDIR)" = "" ]; then \
@ -27,3 +30,7 @@ install:
mkdir -p $(DESTDIR)/etc/kojid
install -p -m 644 kojid.conf $(DESTDIR)/etc/kojid/kojid.conf
for d in $(SUBDIRS); do make DESTDIR=`cd $(DESTDIR); pwd` \
-C $$d install; [ $$? = 0 ] || exit 1; done

View file

@ -26,6 +26,7 @@ except ImportError:
pass
import base64
import koji
import koji.plugin
import koji.util
import commands
import errno
@ -55,6 +56,10 @@ from fnmatch import fnmatch
from optparse import OptionParser
from xmlrpclib import Fault
# our private modules
sys.path.insert(0, '/usr/share/koji-builder/lib')
import tasks
class ServerExit(Exception):
"""Raised to shutdown the server"""
pass
@ -65,6 +70,12 @@ def main():
logger = logging.getLogger("koji.build")
logger.info('Starting up')
tm = TaskManager()
if options.plugin:
#load plugins
pt = koji.plugin.PluginTracker(path=options.pluginpath.split(':'))
for name in options.plugin:
logger.info('Loading plugin: %s' % name)
tm.scanPlugin(pt.load(name))
def shutdown(*args):
raise SystemExit
signal.signal(signal.SIGTERM,shutdown)
@ -544,6 +555,15 @@ class TaskManager(object):
handlers[method] = v
self.handlers = handlers
def scanPlugin(self, plugin):
"""Find task handlers in a plugin"""
# XXX - this is a very simple implementation for now.
# it should be improved
for v in vars(plugin).itervalues():
if type(v) == type(tasks.BaseTaskHandler) and issubclass(v,tasks.BaseTaskHandler):
for method in v.Methods:
self.handlers[method] = v
def shutdown(self):
"""Attempt to shut down cleanly"""
for task_id in self.pids.keys():
@ -1070,7 +1090,11 @@ class TaskManager(object):
handlerClass = self.handlers['default']
else:
raise koji.GenericError, "No handler found for method '%s'" % method
handler = handlerClass(id,method,params)
if issubclass(handlerClass, tasks.BaseTaskHandler):
#new style handler needs session and options passed
handler = handlerClass(id,method,params,session,options)
else:
handler = handlerClass(id,method,params)
# set weight
session.host.setTaskWeight(task_id,handler.weight())
if handler.Foreground:
@ -2615,6 +2639,8 @@ def get_options():
parser.add_option("--topdir", help="Specify topdir")
parser.add_option("--topurl", help="Specify topurl")
parser.add_option("--workdir", help="Specify workdir")
parser.add_option("--pluginpath", help="Specify plugin search path")
parser.add_option("--plugin", action="append", help="Load specified plugin")
parser.add_option("--mockdir", help="Specify mockdir")
parser.add_option("--mockuser", help="User to run mock as")
parser.add_option("-s", "--server", help="url of XMLRPC server")
@ -2639,6 +2665,7 @@ def get_options():
'topdir': '/mnt/koji',
'topurl': None,
'workdir': '/tmp/koji',
'pluginpath': '/usr/lib/koji-builder-plugins',
'mockdir': '/var/lib/mock',
'mockuser': 'kojibuilder',
'packager': 'Koji',
@ -2669,6 +2696,8 @@ def get_options():
defaults[name] = int(value)
except ValueError:
quit("value for %s option must be a valid integer" % name)
elif name in ['plugin', 'plugins']:
defaults['plugin'] = value.split()
elif name in defaults.keys():
defaults[name] = value
else:

20
builder/lib/Makefile Normal file
View file

@ -0,0 +1,20 @@
PYTHON=python
SHAREDIR = $(DESTDIR)/usr/share/koji-builder
MODDIR = $(SHAREDIR)/lib
PYFILES = $(wildcard *.py)
PYVER := $(shell $(PYTHON) -c 'import sys; print "%.3s" %(sys.version)')
_default:
@echo "nothing to make. try make install"
clean:
rm -f *.o *.so *.pyc *~
install:
mkdir -p $(MODDIR)
for p in $(PYFILES) ; do \
install -p -m 644 $$p $(MODDIR)/$$p; \
done
$(PYTHON) -c "import compileall; compileall.compile_dir('$(MODDIR)', 1, '$(PYDIR)', 1)"

224
builder/lib/tasks.py Normal file
View file

@ -0,0 +1,224 @@
# Python module
# tasks handlers for the koji build daemon
# Copyright (c) 2008 Red Hat
#
# Koji is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this software; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors:
# Mike McLean <mikem@redhat.com>
import koji
import logging
import os
import signal
import urllib2
class BaseTaskHandler(object):
"""The base class for task handlers
Each task handler is a class, a new instance of which is created
to handle each task.
"""
# list of methods the class can handle
Methods = []
# Options:
Foreground = False
def __init__(self, id, method, params, session, options, workdir=None):
self.id = id #task id
if method not in self.Methods:
raise koji.GenericError, 'method "%s" is not supported' % method
self.method = method
# handle named parameters
self.params,self.opts = koji.decode_args(*params)
self.session = session
self.options = options
if workdir is None:
workdir = "%s/%s" % (options.workdir, koji.pathinfo.taskrelpath(id))
self.workdir = workdir
self.logger = logging.getLogger("koji.build.BaseTaskHandler")
def setManager(self,manager):
"""Set the manager attribute
This is only used for foreground tasks to give them access
to their task manager.
"""
if not self.Foreground:
return
self.manager = manager
def handler(self):
"""(abstract) the handler for the task."""
raise NotImplementedError
def run(self):
"""Execute the task"""
self.createWorkdir()
try:
return self.handler(*self.params,**self.opts)
finally:
self.removeWorkdir()
_taskWeight = 1.0
def weight(self):
"""Return the weight of the task.
This is run by the taskmanager before the task is run to determine
the weight of the task. The weight is an abstract measure of the
total load the task places on the system while running.
A task may set _taskWeight for a constant weight different from 1, or
override this function for more complicated situations.
Note that task weight is partially ignored while the task is sleeping.
"""
return getattr(self,'_taskWeight',1.0)
def createWorkdir(self):
if self.workdir is None:
return
self.removeWorkdir()
os.makedirs(self.workdir)
def removeWorkdir(self):
if self.workdir is None:
return
safe_rmtree(self.workdir, unmount=False, strict=True)
#os.spawnvp(os.P_WAIT, 'rm', ['rm', '-rf', self.workdir])
def wait(self, subtasks=None, all=False, failany=False):
"""Wait on subtasks
subtasks is a list of integers (or an integer). If more than one subtask
is specified, then the default behavior is to return when any of those
tasks complete. However, if all is set to True, then it waits for all of
them to complete. If all and failany are both set to True, then each
finished task will be checked for failure, and a failure will cause all
of the unfinished tasks to be cancelled.
special values:
subtasks = None specify all subtasks
Implementation notes:
The build daemon forks all tasks as separate processes. This function
uses signal.pause to sleep. The main process watches subtasks in
the database and will send the subprocess corresponding to the
subtask a SIGUSR2 to wake it up when subtasks complete.
"""
if isinstance(subtasks,int):
# allow single integer w/o enclosing list
subtasks = [subtasks]
self.session.host.taskSetWait(self.id,subtasks)
self.logger.debug("Waiting on %r" % subtasks)
while True:
finished, unfinished = self.session.host.taskWait(self.id)
if len(unfinished) == 0:
#all done
break
elif len(finished) > 0:
if all:
if failany:
failed = False
for task in finished:
try:
result = self.session.getTaskResult(task)
except (koji.GenericError, Fault), task_error:
self.logger.info("task %s failed or was canceled" % task)
failed = True
break
if failed:
self.logger.info("at least one task failed or was canceled, cancelling unfinished tasks")
self.session.cancelTaskChildren(self.id)
# reraise the original error now, rather than waiting for
# an error in taskWaitResults()
raise task_error
else:
# at least one done
break
# signal handler set by TaskManager.forkTask
self.logger.debug("Pausing...")
signal.pause()
# main process will wake us up with SIGUSR2
self.logger.debug("...waking up")
self.logger.debug("Finished waiting")
return dict(self.session.host.taskWaitResults(self.id,subtasks))
def getUploadDir(self):
return koji.pathinfo.taskrelpath(self.id)
def uploadFile(self, filename, remoteName=None):
"""Upload the file with the given name to the task output directory
on the hub."""
# Only upload files with content
if os.path.isfile(filename) and os.stat(filename).st_size > 0:
self.session.uploadWrapper(filename, self.getUploadDir(), remoteName)
def localPath(self, relpath):
"""Return a local path to a remote file.
If the file is on an nfs mount, use that, otherwise download a copy"""
if self.options.topurl:
self.logger.debug("Downloading %s", relpath)
url = "%s/%s" % (self.options.topurl, relpath)
fsrc = urllib2.urlopen(url)
fn = "%s/local/%s" % (self.workdir, relpath)
os.makedirs(os.path.dirname(fn))
fdst = file(fn, 'w')
shutil.copyfileobj(fsrc, fdst)
fsrc.close()
fdst.close()
else:
fn = "%s/%s" % (self.options.topdir, relpath)
return fn
#XXX - not the right place for this
#XXX - not as safe as we want
def safe_rmtree(path, unmount=False, strict=True):
logger = logging.getLogger("koji.build")
#safe remove: with -xdev the find cmd will not cross filesystems
# (though it will cross bind mounts from the same filesystem)
if not os.path.exists(path):
logger.debug("No such path: %s" % path)
return
if unmount:
umount_all(path)
#first rm -f non-directories
logger.debug('Scrubbing files in %s' % path)
rv = os.system("find '%s' -xdev \\! -type d -print0 |xargs -0 rm -f" % path)
msg = 'file removal failed (code %r) for %s' % (rv,path)
if rv != 0:
logger.warn(msg)
if strict:
raise koji.GenericError, msg
else:
return rv
#them rmdir directories
#with -depth, we start at the bottom and work up
logger.debug('Scrubbing directories in %s' % path)
rv = os.system("find '%s' -xdev -depth -type d -print0 |xargs -0 rmdir" % path)
msg = 'dir removal failed (code %r) for %s' % (rv,path)
if rv != 0:
logger.warn(msg)
if strict:
raise koji.GenericError, msg
return rv

View file

@ -24,6 +24,7 @@ from ConfigParser import ConfigParser
import sys
import time
import traceback
import types
import pprint
from xmlrpclib import loads,dumps,Fault
from mod_python import apache
@ -31,6 +32,7 @@ from mod_python import apache
import koji
import koji.auth
import koji.db
import koji.plugin
from kojihub import RootExports
from kojihub import HostExports
from koji.context import context
@ -89,6 +91,22 @@ class HandlerRegistry(object):
def register_instance(self,instance):
self.register_module(instance)
def register_plugin(self, plugin):
"""Scan a given plugin for handlers
Handlers are functions marked with one of the decorators defined in koji.plugin
"""
for v in vars(plugin).itervalues():
if isinstance(v, (types.ClassType, types.TypeType)):
#skip classes
continue
if callable(v) and getattr(v, 'exported', False):
if hasattr(v, 'export_alias'):
name = getattr(v, 'export_alias')
else:
name = v.__name__
self.register_function(v, name=name)
def list_api(self):
funcs = []
for name,func in self.funcs.items():
@ -140,6 +158,19 @@ class HandlerRegistry(object):
return func
class HandlerAccess(object):
"""This class is used to grant access to the rpc handlers"""
def __init__(self, registry):
self.__reg = registry
def call(self, __name, *args, **kwargs):
return self.__reg.get(__name)(*args, **kwargs)
def get(self, name):
return self.__Reg.get(name)
class ModXMLRPCRequestHandler(object):
"""Simple XML-RPC handler for mod_python environment"""
@ -194,6 +225,7 @@ class ModXMLRPCRequestHandler(object):
faultString = "%s: %s" % (e_class,e)
sys.stderr.write(tb_str)
sys.stderr.write('\n')
sys.stderr.flush()
response = dumps(Fault(faultCode, faultString))
if _opt_bool(context.opts, 'KojiDebug'):
@ -337,6 +369,9 @@ def load_config(req):
['KojiWebURL', 'string', 'http://localhost.localdomain/koji'],
['EmailDomain', 'string', None],
['Plugins', 'string', None],
['PluginPath', 'string', '/usr/lib/koji-hub-plugins'],
['KojiDebug', 'boolean', False],
['KojiTraceback', 'string', None],
['EnableFunctionDebug', 'boolean', False],
@ -375,19 +410,44 @@ def load_config(req):
return opts
def load_plugins(opts):
"""Load plugins specified by our configuration"""
if not opts['Plugins']:
return
tracker = koji.plugin.PluginTracker(path=opts['PluginPath'].split(':'))
for name in opts['Plugins'].split():
sys.stderr.write('Loading plugin: %s\n' % name)
try:
tracker.load(name)
except Exception:
sys.stderr.write(''.join(traceback.format_exception(*sys.exc_info())))
#make this non-fatal, but set ServerOffline
opts['ServerOffline'] = True
opts['OfflineMessage'] = 'configuration error'
sys.stderr.flush()
return tracker
#
# mod_python handler
#
firstcall = True
ready = False
opts = {}
def handler(req, profiling=False):
global firstcall, registry, opts
global firstcall, ready, registry, opts, plugins
if firstcall:
registry = get_registry()
opts = load_config(req)
firstcall = False
opts = load_config(req)
plugins = load_plugins(opts)
registry = get_registry(opts, plugins)
ready = True
if not ready:
#this will happen on subsequent passes if an error happens in the firstcall code
opts['ServerOffline'] = True
opts['OfflineMessage'] = 'server startup error'
if profiling:
import profile, pstats, StringIO, tempfile
global _profiling_req
@ -409,6 +469,7 @@ def handler(req, profiling=False):
context._threadclear()
context.commit_pending = False
context.opts = opts
context.handlers = HandlerAccess(registry)
context.req = req
koji.db.provideDBopts(database = opts["DBName"],
user = opts["DBUser"],
@ -433,7 +494,7 @@ def handler(req, profiling=False):
context._threadclear()
return apache.OK
def get_registry():
def get_registry(opts, plugins):
# Create and populate handler registry
registry = HandlerRegistry()
functions = RootExports()
@ -448,5 +509,7 @@ def get_registry():
registry.register_function(koji.auth.logoutChild)
registry.register_function(koji.auth.exclusiveSession)
registry.register_function(koji.auth.sharedSession)
for name in opts.get('Plugins', '').split():
registry.register_plugin(plugins.get(name))
return registry

View file

@ -136,6 +136,7 @@ rm -rf $RPM_BUILD_ROOT
%config(noreplace) %{_sysconfdir}/sysconfig/kojid
%{_sysconfdir}/kojid
%config(noreplace) %{_sysconfdir}/kojid/kojid.conf
%{_datadir}/koji-builder
%attr(-,kojibuilder,kojibuilder) /etc/mock/koji
%pre builder

103
koji/plugin.py Normal file
View file

@ -0,0 +1,103 @@
# koji plugin module
# Copyright (c) 2008 Red Hat
#
# Koji is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation;
# version 2.1 of the License.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this software; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#
# Authors:
# Mike McLean <mikem@redhat.com>
import imp
import koji
import sys
class PluginTracker(object):
def __init__(self, path=None, prefix='_koji_plugin__'):
self.searchpath = path
#prefix should not have a '.' in it, this can cause problems.
self.prefix = prefix
self.plugins = {}
def load(self, name, path=None, reload=False):
if self.plugins.has_key(name) and not reload:
return self.plugins[name]
mod_name = name
if self.prefix:
#mod_name determines how the module is named in sys.modules
#Using a prefix helps prevent overlap with other modules
#(no '.' -- it causes problems)
mod_name = self.prefix + name
if sys.modules.has_key(mod_name) and not reload:
raise koji.GenericError, 'module name conflict: %s' % mod_name
if path is None:
path = self.searchpath
if path is None:
raise koji.GenericError, "empty module search path"
file, pathname, description = imp.find_module(name, self.pathlist(path))
try:
plugin = imp.load_module(mod_name, file, pathname, description)
finally:
file.close()
self.plugins[name] = plugin
return plugin
def get(self, name):
return self.plugins.get(name)
def pathlist(self, path):
if isinstance(path, basestring):
return [path]
else:
return path
# some decorators used by plugins
def export(f):
"""a decorator that marks a function as exported
intended to be used by plugins
the HandlerRegistry will export the function under its own name
"""
setattr(f, 'exported', True)
return f
def export_as(alias):
"""returns a decorator that marks a function as exported and gives it an alias
indended to be used by plugins
"""
def dec(f):
setattr(f, 'exported', True)
setattr(f, 'export_alias', alias)
return f
return dec
def export_in(module, alias=None):
"""returns a decorator that marks a function as exported with a module prepended
optionally, can also alias the function within the module
indended to be used by plugins
"""
def dec(f):
if alias is None:
alias = "%s.%s" % (module, f.__name__)
else:
alias = "%s.%s" % (module, alias)
setattr(f, 'exported', True)
setattr(f, 'export_module', module)
setattr(f, 'export_alias', alias)
return f
return dec