Install both hub and builder plugins.
This is a different take at https://pagure.io/koji/pull-request/45 Instead of installing just the runroot builder plugin explicitly, here we separate the plugins out into hub and builder plugins explicitly and install each type in turn.
This commit is contained in:
Ralph Bean
parent
79721c4dc9
commit
43ba5a6071
9 changed files with 11 additions and 2 deletions
15
plugins/hub/echo.py
Normal file
15
plugins/hub/echo.py
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
# Example Koji callback
|
||||
# Copyright (c) 2009-2014 Red Hat, Inc.
|
||||
# This callback simply logs all of its args using the logging module
|
||||
#
|
||||
# Authors:
|
||||
# Mike Bonnet <mikeb@redhat.com>
|
||||
|
||||
from koji.plugin import callbacks, callback, ignore_error
|
||||
import logging
|
||||
|
||||
@callback(*callbacks.keys())
|
||||
@ignore_error
|
||||
def echo(cbtype, *args, **kws):
|
||||
logging.getLogger('koji.plugin.echo').info('Called the %s callback, args: %s; kws: %s',
|
||||
cbtype, str(args), str(kws))
|
||||
24
plugins/hub/messagebus.conf
Normal file
24
plugins/hub/messagebus.conf
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
# config file for the Koji messagebus plugin
|
||||
|
||||
[broker]
|
||||
host = amqp.example.com
|
||||
port = 5671
|
||||
ssl = true
|
||||
timeout = 10
|
||||
heartbeat = 60
|
||||
# PLAIN options
|
||||
auth = PLAIN
|
||||
username = guest
|
||||
password = guest
|
||||
# GSSAPI options
|
||||
# auth = GSSAPI
|
||||
# keytab = /etc/koji-hub/plugins/koji-messagebus.keytab
|
||||
# principal = messagebus/koji.example.com@EXAMPLE.COM
|
||||
|
||||
[exchange]
|
||||
name = koji.events
|
||||
type = topic
|
||||
durable = true
|
||||
|
||||
[topic]
|
||||
prefix = koji.event
|
||||
226
plugins/hub/messagebus.py
Normal file
226
plugins/hub/messagebus.py
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
# Koji callback for sending notifications about events to a messagebus (amqp broker)
|
||||
# Copyright (c) 2009-2014 Red Hat, Inc.
|
||||
#
|
||||
# Authors:
|
||||
# Mike Bonnet <mikeb@redhat.com>
|
||||
|
||||
from koji.plugin import callbacks, callback, ignore_error
|
||||
import ConfigParser
|
||||
import logging
|
||||
import qpid.messaging
|
||||
import qpid.messaging.transports
|
||||
from ssl import wrap_socket
|
||||
import socket
|
||||
import os
|
||||
import krbV
|
||||
|
||||
MAX_KEY_LENGTH = 255
|
||||
CONFIG_FILE = '/etc/koji-hub/plugins/messagebus.conf'
|
||||
|
||||
config = None
|
||||
session = None
|
||||
target = None
|
||||
|
||||
def connect_timeout(host, port, timeout):
|
||||
for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
|
||||
af, socktype, proto, canonname, sa = res
|
||||
sock = socket.socket(af, socktype, proto)
|
||||
sock.settimeout(timeout)
|
||||
try:
|
||||
sock.connect(sa)
|
||||
break
|
||||
except socket.error, msg:
|
||||
sock.close()
|
||||
else:
|
||||
# If we got here then we couldn't connect (yet)
|
||||
raise
|
||||
return sock
|
||||
|
||||
class tlstimeout(qpid.messaging.transports.tls):
|
||||
def __init__(self, conn, host, port):
|
||||
self.socket = connect_timeout(host, port, getattr(conn, '_timeout'))
|
||||
if conn.tcp_nodelay:
|
||||
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
self.tls = wrap_socket(self.socket, keyfile=conn.ssl_keyfile, certfile=conn.ssl_certfile, ca_certs=conn.ssl_trustfile)
|
||||
self.socket.setblocking(0)
|
||||
self.state = None
|
||||
|
||||
qpid.messaging.transports.TRANSPORTS['tls+timeout'] = tlstimeout
|
||||
|
||||
class Connection(qpid.messaging.Connection):
|
||||
"""
|
||||
A connection class which supports a timeout option
|
||||
to the establish() method. Only necessary until
|
||||
upstream Apache Qpid commit 1487578 is available in
|
||||
a supported release.
|
||||
"""
|
||||
@staticmethod
|
||||
def establish(url=None, timeout=None, **options):
|
||||
conn = Connection(url, **options)
|
||||
conn._timeout = timeout
|
||||
conn.open()
|
||||
return conn
|
||||
|
||||
def _wait(self, predicate, timeout=None):
|
||||
if timeout is None and hasattr(self, '_timeout'):
|
||||
timeout = self._timeout
|
||||
return qpid.messaging.Connection._wait(self, predicate, timeout)
|
||||
|
||||
def get_sender():
|
||||
global config, session, target
|
||||
if session and target:
|
||||
try:
|
||||
return session.sender(target)
|
||||
except:
|
||||
logging.getLogger('koji.plugin.messagebus').warning('Error getting session, will retry', exc_info=True)
|
||||
session = None
|
||||
target = None
|
||||
|
||||
config = ConfigParser.SafeConfigParser()
|
||||
config.read(CONFIG_FILE)
|
||||
if not config.has_option('broker', 'timeout'):
|
||||
config.set('broker', 'timeout', '60')
|
||||
if not config.has_option('broker', 'heartbeat'):
|
||||
config.set('broker', 'heartbeat', '60')
|
||||
|
||||
if config.getboolean('broker', 'ssl'):
|
||||
url = 'amqps://'
|
||||
else:
|
||||
url = 'amqp://'
|
||||
auth = config.get('broker', 'auth')
|
||||
if auth == 'PLAIN':
|
||||
url += config.get('broker', 'username') + '/'
|
||||
url += config.get('broker', 'password') + '@'
|
||||
elif auth == 'GSSAPI':
|
||||
ccname = 'MEMORY:messagebus'
|
||||
os.environ['KRB5CCNAME'] = ccname
|
||||
ctx = krbV.default_context()
|
||||
ccache = krbV.CCache(name=ccname, context=ctx)
|
||||
cprinc = krbV.Principal(name=config.get('broker', 'principal'), context=ctx)
|
||||
ccache.init(principal=cprinc)
|
||||
keytab = krbV.Keytab(name='FILE:' + config.get('broker', 'keytab'), context=ctx)
|
||||
ccache.init_creds_keytab(principal=cprinc, keytab=keytab)
|
||||
else:
|
||||
raise koji.PluginError, 'unsupported auth type: %s' % auth
|
||||
|
||||
url += config.get('broker', 'host') + ':'
|
||||
url += config.get('broker', 'port')
|
||||
|
||||
conn = Connection.establish(url,
|
||||
sasl_mechanisms=config.get('broker', 'auth'),
|
||||
transport='tls+timeout',
|
||||
timeout=config.getfloat('broker', 'timeout'),
|
||||
heartbeat=config.getint('broker', 'heartbeat'))
|
||||
sess = conn.session()
|
||||
tgt = """%s;
|
||||
{ create: sender,
|
||||
assert: always,
|
||||
node: { type: topic,
|
||||
durable: %s,
|
||||
x-declare: { exchange: "%s",
|
||||
type: %s } } }""" % \
|
||||
(config.get('exchange', 'name'), config.getboolean('exchange', 'durable'),
|
||||
config.get('exchange', 'name'), config.get('exchange', 'type'))
|
||||
sender = sess.sender(tgt)
|
||||
session = sess
|
||||
target = tgt
|
||||
|
||||
return sender
|
||||
|
||||
def _token_append(tokenlist, val):
|
||||
# Replace any periods with underscores so we have a deterministic number of tokens
|
||||
val = val.replace('.', '_')
|
||||
tokenlist.append(val)
|
||||
|
||||
def get_message_subject(msgtype, *args, **kws):
|
||||
key = [config.get('topic', 'prefix'), msgtype]
|
||||
|
||||
if msgtype == 'PackageListChange':
|
||||
_token_append(key, kws['tag']['name'])
|
||||
_token_append(key, kws['package']['name'])
|
||||
elif msgtype == 'TaskStateChange':
|
||||
_token_append(key, kws['info']['method'])
|
||||
_token_append(key, kws['attribute'])
|
||||
elif msgtype == 'BuildStateChange':
|
||||
info = kws['info']
|
||||
_token_append(key, kws['attribute'])
|
||||
_token_append(key, info['name'])
|
||||
elif msgtype == 'Import':
|
||||
_token_append(key, kws['type'])
|
||||
elif msgtype in ('Tag', 'Untag'):
|
||||
_token_append(key, kws['tag']['name'])
|
||||
build = kws['build']
|
||||
_token_append(key, build['name'])
|
||||
_token_append(key, kws['user']['name'])
|
||||
elif msgtype == 'RepoInit':
|
||||
_token_append(key, kws['tag']['name'])
|
||||
elif msgtype == 'RepoDone':
|
||||
_token_append(key, kws['repo']['tag_name'])
|
||||
|
||||
key = '.'.join(key)
|
||||
key = key[:MAX_KEY_LENGTH]
|
||||
return key
|
||||
|
||||
def get_message_headers(msgtype, *args, **kws):
|
||||
headers = {'type': msgtype}
|
||||
|
||||
if msgtype == 'PackageListChange':
|
||||
headers['tag'] = kws['tag']['name']
|
||||
headers['package'] = kws['package']['name']
|
||||
elif msgtype == 'TaskStateChange':
|
||||
headers['id'] = kws['info']['id']
|
||||
headers['parent'] = kws['info']['parent']
|
||||
headers['method'] = kws['info']['method']
|
||||
headers['attribute'] = kws['attribute']
|
||||
headers['old'] = kws['old']
|
||||
headers['new'] = kws['new']
|
||||
elif msgtype == 'BuildStateChange':
|
||||
info = kws['info']
|
||||
headers['name'] = info['name']
|
||||
headers['version'] = info['version']
|
||||
headers['release'] = info['release']
|
||||
headers['attribute'] = kws['attribute']
|
||||
headers['old'] = kws['old']
|
||||
headers['new'] = kws['new']
|
||||
elif msgtype == 'Import':
|
||||
headers['importType'] = kws['type']
|
||||
elif msgtype in ('Tag', 'Untag'):
|
||||
headers['tag'] = kws['tag']['name']
|
||||
build = kws['build']
|
||||
headers['name'] = build['name']
|
||||
headers['version'] = build['version']
|
||||
headers['release'] = build['release']
|
||||
headers['user'] = kws['user']['name']
|
||||
elif msgtype == 'RepoInit':
|
||||
headers['tag'] = kws['tag']['name']
|
||||
elif msgtype == 'RepoDone':
|
||||
headers['tag'] = kws['repo']['tag_name']
|
||||
|
||||
return headers
|
||||
|
||||
@callback(*[c for c in callbacks.keys() if c.startswith('post')])
|
||||
@ignore_error
|
||||
def send_message(cbtype, *args, **kws):
|
||||
global config
|
||||
sender = get_sender()
|
||||
if cbtype.startswith('post'):
|
||||
msgtype = cbtype[4:]
|
||||
else:
|
||||
msgtype = cbtype[3:]
|
||||
|
||||
data = kws.copy()
|
||||
if args:
|
||||
data['args'] = list(args)
|
||||
|
||||
exchange_type = config.get('exchange', 'type')
|
||||
if exchange_type == 'topic':
|
||||
subject = get_message_subject(msgtype, *args, **kws)
|
||||
message = qpid.messaging.Message(subject=subject, content=data)
|
||||
elif exchange_type == 'headers':
|
||||
headers = get_message_headers(msgtype, *args, **kws)
|
||||
message = qpid.messaging.Message(properties=headers, content=data)
|
||||
else:
|
||||
raise koji.PluginError, 'unsupported exchange type: %s' % exchange_type
|
||||
|
||||
sender.send(message, sync=True, timeout=config.getfloat('broker', 'timeout'))
|
||||
sender.close(timeout=config.getfloat('broker', 'timeout'))
|
||||
5
plugins/hub/rpm2maven.conf
Normal file
5
plugins/hub/rpm2maven.conf
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
# config file for the Koji rpm2maven plugin
|
||||
|
||||
[patterns]
|
||||
rpm_names = *-repolib
|
||||
artifact_paths = /usr/share/java/repository/maven2/*
|
||||
107
plugins/hub/rpm2maven.py
Normal file
107
plugins/hub/rpm2maven.py
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
# Koji callback for extracting Maven artifacts (.pom and .jar files)
|
||||
# from an rpm and making them available via the Koji-managed Maven repo.
|
||||
# Copyright (c) 2010-2014 Red Hat, Inc.
|
||||
#
|
||||
# Authors:
|
||||
# Mike Bonnet <mikeb@redhat.com>
|
||||
|
||||
import koji
|
||||
from koji.context import context
|
||||
from koji.plugin import callback
|
||||
import ConfigParser
|
||||
import fnmatch
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
|
||||
CONFIG_FILE = '/etc/koji-hub/plugins/rpm2maven.conf'
|
||||
|
||||
config = None
|
||||
|
||||
@callback('postImport')
|
||||
def maven_import(cbtype, *args, **kws):
|
||||
global config
|
||||
if not context.opts.get('EnableMaven', False):
|
||||
return
|
||||
if kws.get('type') != 'rpm':
|
||||
return
|
||||
buildinfo = kws['build']
|
||||
rpminfo = kws['rpm']
|
||||
filepath = kws['filepath']
|
||||
|
||||
if not config:
|
||||
config = ConfigParser.SafeConfigParser()
|
||||
config.read(CONFIG_FILE)
|
||||
name_patterns = config.get('patterns', 'rpm_names').split()
|
||||
for pattern in name_patterns:
|
||||
if fnmatch.fnmatch(rpminfo['name'], pattern):
|
||||
break
|
||||
else:
|
||||
return
|
||||
|
||||
tmpdir = os.path.join(koji.pathinfo.work(), 'rpm2maven', koji.buildLabel(buildinfo))
|
||||
try:
|
||||
if os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir)
|
||||
koji.ensuredir(tmpdir)
|
||||
expand_rpm(filepath, tmpdir)
|
||||
scan_and_import(buildinfo, rpminfo, tmpdir)
|
||||
finally:
|
||||
if os.path.exists(tmpdir):
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
def expand_rpm(filepath, tmpdir):
|
||||
devnull = file('/dev/null', 'r+')
|
||||
rpm2cpio = subprocess.Popen(['/usr/bin/rpm2cpio', filepath],
|
||||
stdout=subprocess.PIPE,
|
||||
stdin=devnull, stderr=devnull,
|
||||
close_fds=True)
|
||||
cpio = subprocess.Popen(['/bin/cpio', '-id'],
|
||||
stdin=rpm2cpio.stdout,
|
||||
cwd=tmpdir,
|
||||
stdout=devnull, stderr=devnull,
|
||||
close_fds=True)
|
||||
if rpm2cpio.wait() != 0 or cpio.wait() != 0:
|
||||
raise koji.CallbackError, 'error extracting files from %s, ' \
|
||||
'rpm2cpio returned %s, cpio returned %s' % \
|
||||
(filepath, rpm2cpio.wait(), cpio.wait())
|
||||
devnull.close()
|
||||
|
||||
def scan_and_import(buildinfo, rpminfo, tmpdir):
|
||||
global config
|
||||
path_patterns = config.get('patterns', 'artifact_paths').split()
|
||||
|
||||
maven_archives = []
|
||||
for dirpath, dirnames, filenames in os.walk(tmpdir):
|
||||
relpath = dirpath[len(tmpdir):]
|
||||
for pattern in path_patterns:
|
||||
if fnmatch.fnmatch(relpath, pattern):
|
||||
break
|
||||
else:
|
||||
continue
|
||||
|
||||
poms = [f for f in filenames if f.endswith('.pom')]
|
||||
if len(poms) != 1:
|
||||
continue
|
||||
|
||||
pom_info = koji.parse_pom(os.path.join(dirpath, poms[0]))
|
||||
maven_info = koji.pom_to_maven_info(pom_info)
|
||||
maven_archives.append({'maven_info': maven_info,
|
||||
'files': [os.path.join(dirpath, f) for f in filenames]})
|
||||
|
||||
if not maven_archives:
|
||||
return
|
||||
|
||||
# We don't know which pom is the top-level pom, so we don't know what Maven
|
||||
# metadata to associate with the build. So we make something up.
|
||||
maven_build = {'group_id': buildinfo['name'], 'artifact_id': rpminfo['name'],
|
||||
'version': '%(version)s-%(release)s' % buildinfo}
|
||||
context.handlers.call('host.createMavenBuild', buildinfo, maven_build)
|
||||
|
||||
for entry in maven_archives:
|
||||
maven_info = entry['maven_info']
|
||||
for filepath in entry['files']:
|
||||
if not context.handlers.call('getArchiveType', filename=filepath):
|
||||
# unsupported archive type, skip it
|
||||
continue
|
||||
context.handlers.call('host.importArchive', filepath, buildinfo, 'maven', maven_info)
|
||||
61
plugins/hub/runroot_hub.py
Normal file
61
plugins/hub/runroot_hub.py
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
#koji hub plugin
|
||||
# There is a kojid plugin that goes with this hub plugin. The kojid builder
|
||||
# plugin has a config file. This hub plugin has no config file.
|
||||
|
||||
|
||||
from koji.context import context
|
||||
from koji.plugin import export
|
||||
import koji
|
||||
import random
|
||||
import sys
|
||||
|
||||
#XXX - have to import kojihub for mktask
|
||||
sys.path.insert(0, '/usr/share/koji-hub/')
|
||||
from kojihub import mktask, get_tag, get_all_arches
|
||||
|
||||
__all__ = ('runroot',)
|
||||
|
||||
|
||||
def get_channel_arches(channel):
|
||||
"""determine arches available in channel"""
|
||||
chan = context.handlers.call('getChannel', channel, strict=True)
|
||||
ret = {}
|
||||
for host in context.handlers.call('listHosts', channelID=chan['id'], enabled=True):
|
||||
for a in host['arches'].split():
|
||||
ret[koji.canonArch(a)] = 1
|
||||
return ret
|
||||
|
||||
@export
|
||||
def runroot(tagInfo, arch, command, channel=None, **opts):
|
||||
""" Create a runroot task """
|
||||
context.session.assertPerm('runroot')
|
||||
taskopts = {
|
||||
'priority': 15,
|
||||
'arch': arch,
|
||||
}
|
||||
|
||||
taskopts['channel'] = channel or 'runroot'
|
||||
|
||||
if arch == 'noarch':
|
||||
#not all arches can generate a proper buildroot for all tags
|
||||
tag = get_tag(tagInfo)
|
||||
if not tag['arches']:
|
||||
raise koji.GenericError, 'no arches defined for tag %s' % tag['name']
|
||||
|
||||
#get all known arches for the system
|
||||
fullarches = get_all_arches()
|
||||
|
||||
tagarches = tag['arches'].split()
|
||||
|
||||
# If our tag can't do all arches, then we need to
|
||||
# specify one of the arches it can do.
|
||||
if set(fullarches) - set(tagarches):
|
||||
chanarches = get_channel_arches(taskopts['channel'])
|
||||
choices = [x for x in tagarches if x in chanarches]
|
||||
if not choices:
|
||||
raise koji.GenericError, 'no common arches for tag/channel: %s/%s' \
|
||||
% (tagInfo, taskopts['channel'])
|
||||
taskopts['arch'] = koji.canonArch(random.choice(choices))
|
||||
|
||||
return mktask(taskopts,'runroot', tagInfo, arch, command, **opts)
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue