rpm2maven: a plugin to extract Maven artifacts from a rpm build and make them available in the Koji-managed Maven repo

hub/kojihub.py

@@ -4100,7 +4100,8 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False):
     rpminfo['payloadhash'] = koji.hex_string(hdr[rpm.RPMTAG_SIGMD5])
     rpminfo['brootid'] = brootid
 
-    koji.plugin.run_callbacks('preImport', type='rpm', rpm=rpminfo, build=buildinfo)
+    koji.plugin.run_callbacks('preImport', type='rpm', rpm=rpminfo, build=buildinfo,
+                              filepath=fn)
 
     q = """INSERT INTO rpminfo (id,name,version,release,epoch,
             build_id,arch,buildtime,buildroot_id,
@@ -4113,7 +4114,8 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False):
     """
     _dml(q, rpminfo)
 
-    koji.plugin.run_callbacks('postImport', type='rpm', rpm=rpminfo, build=buildinfo)
+    koji.plugin.run_callbacks('postImport', type='rpm', rpm=rpminfo, build=buildinfo,
+                              filepath=fn)
 
     return rpminfo
 
@@ -4346,19 +4348,10 @@ def get_archive_type(filename=None, type_name=None, type_id=None, strict=False):
 def new_maven_build(build, maven_info):
     """
     Add Maven metadata to an existing build.
-    maven_info must contain the 'group_id' and
-    'artifact_id' keys.
-
-    Note: the "group_id-artifact_id" must match the name of the build
+    maven_info must contain the 'group_id',
+    'artifact_id', and 'version' keys.
     """
     maven_info = maven_info.copy()
-    maven_nvr = koji.maven_info_to_nvr(maven_info)
-    maven_nvr['release'] = build['release']
-
-    for field in ('name', 'version', 'epoch'):
-        if build[field] != maven_nvr[field]:
-            raise koji.BuildError, '%s mismatch (build: %s, maven: %s)' % \
-                (field, build[field], maven_nvr[field])
 
     current_maven_info = get_maven_build(build)
     if current_maven_info:
@@ -4424,7 +4417,7 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None):
     archiveinfo['md5sum'] = m.hexdigest()
 
     koji.plugin.run_callbacks('preImport', type='archive', archive=archiveinfo, build=buildinfo,
-                              build_type=type)
+                              build_type=type, filepath=filepath)
 
     # XXX verify that the buildroot is associated with a task that's associated with the build
     archive_id = _singleValue("SELECT nextval('archiveinfo_id_seq')", strict=True)
@@ -4434,7 +4427,6 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None):
 
     if type == 'maven':
         maveninfo = get_maven_build(buildinfo, strict=True)
-        buildinfo.update(dslice(maveninfo, ('group_id', 'artifact_id', 'version')))
 
         if archivetype['name'] == 'pom':
             pom_info = koji.parse_pom(filepath)
@@ -4454,7 +4446,6 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None):
         _generate_maven_metadata(maveninfo, mavendir)
     elif type == 'win':
         wininfo = get_win_build(buildinfo, strict=True)
-        buildinfo['platform'] = wininfo['platform']
 
         insert = InsertProcessor('win_archives')
         insert.set(archive_id=archive_id)
@@ -4476,7 +4467,7 @@ def import_archive(filepath, buildinfo, type, typeInfo, buildroot_id=None):
 
     archiveinfo = get_archive(archive_id, strict=True)
     koji.plugin.run_callbacks('postImport', type='archive', archive=archiveinfo, build=buildinfo,
-                              build_type=type)
+                              build_type=type, filepath=filepath)
 
 def _import_archive_file(filepath, destdir):
     """
@@ -6272,7 +6263,12 @@ def importImageInternal(task_id, filename, filesize, arch, mediatype, hash, rpml
     imageinfo['hash'] = hash
     # TODO: add xmlfile field to the imageinfo table
 
-    koji.plugin.run_callbacks('preImport', type='image', image=imageinfo)
+    filepath = os.path.join(koji.pathinfo.work(),
+                            koji.pathinfo.taskrelpath(task_id),
+                            filename)
+
+    koji.plugin.run_callbacks('preImport', type='image', image=imageinfo,
+                              filepath=filepath)
 
     q = """INSERT INTO imageinfo (id,task_id,filename,filesize,
            arch,mediatype,hash)
@@ -6297,7 +6293,8 @@ def importImageInternal(task_id, filename, filesize, arch, mediatype, hash, rpml
     for rpm_id in rpm_ids:
         _dml(q, locals())
 
-    koji.plugin.run_callbacks('postImport', type='image', image=imageinfo)
+    koji.plugin.run_callbacks('postImport', type='image', image=imageinfo,
+                              filepath=filepath)
 
     return image_id
 
@@ -6846,7 +6843,7 @@ class RootExports(object):
         filepath: path to the archive file (relative to the Koji workdir)
         buildinfo: information about the build to associate the archive with
                    May be a string (NVR), integer (buildID), or dict (containing keys: name, version, release)
-        type: type of the archive being imported.  Currently supported archive types: maven
+        type: type of the archive being imported.  Currently supported archive types: maven, win
         typeInfo: dict of type-specific information
         """
         if type == 'maven':
@@ -9287,6 +9284,17 @@ class HostExports(object):
 
         return data
 
+    def createMavenBuild(self, build_info, maven_info):
+        """
+        Associate Maven metadata with an existing build.  Used
+        by the rpm2maven plugin.
+        """
+        host = Host()
+        host.verify()
+        if not context.opts.get('EnableMaven'):
+            raise koji.GenericError, "Maven support not enabled"
+        new_maven_build(build_info, maven_info)
+
     def completeMavenBuild(self, task_id, build_id, maven_results, rpm_results):
         """Complete the Maven build."""
         if not context.opts.get('EnableMaven'):
@@ -9343,6 +9351,23 @@ class HostExports(object):
         # send email
         build_notification(task_id, build_id)
 
+    def importArchive(self, filepath, buildinfo, type, typeInfo):
+        """
+        Import an archive file and associate it with a build.  The archive can
+        be any non-rpm filetype supported by Koji.  Used by the rpm2maven plugin.
+        """
+        host = Host()
+        host.verify()
+        if type == 'maven':
+            if not context.opts.get('EnableMaven'):
+                raise koji.GenericError, 'Maven support not enabled'
+        elif type == 'win':
+            if not context.opts.get('EnableWin'):
+                raise koji.GenericError, 'Windows support not enabled'
+        else:
+            koji.GenericError, 'unsupported archive type: %s' % type
+        import_archive(filepath, buildinfo, type, typeInfo)
+
     def importWrapperRPMs(self, task_id, build_id, rpm_results):
         """Import the wrapper rpms and associate them with the given build.  Any existing
            rpms are deleted before import."""

koji.spec

@@ -51,6 +51,7 @@ Requires: python-qpid >= 0.7
 %if 0%{?rhel} == 5
 Requires: python-ssl
 %endif
+Requires: cpio
 
 %description hub-plugins
 Plugins to the koji XMLRPC interface

plugins/rpm2maven.conf

@@ -0,0 +1,5 @@
+# config file for the Koji rpm2maven plugin
+
+[patterns]
+rpm_names = *-repolib
+artifact_paths = /usr/share/java/repository/maven2/*

plugins/rpm2maven.py

@@ -0,0 +1,107 @@
+# Koji callback for extracting Maven artifacts (.pom and .jar files)
+# from an rpm and making them available via the Koji-managed Maven repo.
+# Copyright (c) 2010 Red Hat, Inc.
+#
+# Authors:
+#     Mike Bonnet <mikeb@redhat.com>
+
+import koji
+from koji.context import context
+from koji.plugin import callback
+import ConfigParser
+import fnmatch
+import os
+import shutil
+import subprocess
+
+CONFIG_FILE = '/etc/koji-hub/plugins/rpm2maven.conf'
+
+config = None
+
+@callback('postImport')
+def maven_import(cbtype, *args, **kws):
+    global config
+    if not context.opts.get('EnableMaven', False):
+        return
+    if kws.get('type') != 'rpm':
+        return
+    buildinfo = kws['build']
+    rpminfo = kws['rpm']
+    filepath = kws['filepath']
+
+    if not config:
+        config = ConfigParser.SafeConfigParser()
+        config.read(CONFIG_FILE)
+    name_patterns = config.get('patterns', 'rpm_names').split()
+    for pattern in name_patterns:
+        if fnmatch.fnmatch(rpminfo['name'], pattern):
+            break
+    else:
+        return
+
+    tmpdir = os.path.join(koji.pathinfo.work(), 'rpm2maven', koji.buildLabel(buildinfo))
+    try:
+        if os.path.exists(tmpdir):
+            shutil.rmtree(tmpdir)
+        koji.ensuredir(tmpdir)
+        expand_rpm(filepath, tmpdir)
+        scan_and_import(buildinfo, rpminfo, tmpdir)
+    finally:
+        if os.path.exists(tmpdir):
+            shutil.rmtree(tmpdir)
+
+def expand_rpm(filepath, tmpdir):
+    devnull = file('/dev/null', 'r+')
+    rpm2cpio = subprocess.Popen(['/usr/bin/rpm2cpio', filepath],
+                                stdout=subprocess.PIPE,
+                                stdin=devnull, stderr=devnull,
+                                close_fds=True)
+    cpio = subprocess.Popen(['/bin/cpio', '-id'],
+                            stdin=rpm2cpio.stdout,
+                            cwd=tmpdir,
+                            stdout=devnull, stderr=devnull,
+                            close_fds=True)
+    if rpm2cpio.wait() != 0 or cpio.wait() != 0:
+        raise koji.CallbackError, 'error extracting files from %s, ' \
+              'rpm2cpio returned %s, cpio returned %s' % \
+              (filepath, rpm2cpio.wait(), cpio.wait())
+    devnull.close()
+
+def scan_and_import(buildinfo, rpminfo, tmpdir):
+    global config
+    path_patterns = config.get('patterns', 'artifact_paths').split()
+
+    maven_archives = []
+    for dirpath, dirnames, filenames in os.walk(tmpdir):
+        relpath = dirpath[len(tmpdir):]
+        for pattern in path_patterns:
+            if fnmatch.fnmatch(relpath, pattern):
+                break
+        else:
+            continue
+
+        poms = [f for f in filenames if f.endswith('.pom')]
+        if len(poms) != 1:
+            continue
+
+        pom_info = koji.parse_pom(os.path.join(dirpath, poms[0]))
+        maven_info = koji.pom_to_maven_info(pom_info)
+        maven_archives.append({'maven_info': maven_info,
+                               'files': [os.path.join(dirpath, f) for f in filenames]})
+
+    if not maven_archives:
+        return
+
+    # We don't know which pom is the top-level pom, so we don't know what Maven
+    # metadata to associate with the build.  So we make something up.
+    maven_build = {'group_id': buildinfo['name'], 'artifact_id': rpminfo['name'],
+                   'version': '%(version)s-%(release)s' % buildinfo}
+    context.handlers.call('host.createMavenBuild', buildinfo, maven_build)
+
+    for entry in maven_archives:
+        maven_info = entry['maven_info']
+        for filepath in entry['files']:
+            if not context.handlers.call('getArchiveType', filename=filepath):
+                # unsupported archive type, skip it
+                continue
+            context.handlers.call('host.importArchive', filepath, buildinfo, 'maven', maven_info)