support for including external repo data in Koji repos

builder/Makefile

@@ -1,6 +1,7 @@
 
 SUBDIRS = lib
 BINFILES = kojid
+LIBEXECFILES = mergerepos
 PYFILES = $(wildcard *.py)
 
 _default:
@@ -21,6 +22,9 @@ install:
 	mkdir -p $(DESTDIR)/usr/sbin
 	install -p -m 755 $(BINFILES) $(DESTDIR)/usr/sbin
 
+	mkdir -p $(DESTDIR)/usr/libexec/kojid
+	install -p -m 755 $(LIBEXECFILES) $(DESTDIR)/usr/libexec/kojid
+
 	mkdir -p $(DESTDIR)/etc/mock/koji
 	mkdir -p $(DESTDIR)/etc/rc.d/init.d
 	install -p -m 755 kojid.init $(DESTDIR)/etc/rc.d/init.d/kojid

builder/kojid

@@ -1696,9 +1696,8 @@ class BuildTask(BaseTaskHandler):
 
     def getSRPMFromSCM(self, url, build_tag, repo_id):
         #TODO - allow different ways to get the srpm
-        build_tag_info = session.getTag(build_tag, strict=True)
         task_id = session.host.subtask(method='buildSRPMFromSCM',
-                                       arglist=[url, build_tag_info, {'repo_id': repo_id}],
+                                       arglist=[url, build_tag, repo_id],
                                        label='srpm',
                                        parent=self.id)
         # wait for subtask to finish
@@ -2003,19 +2002,22 @@ class BuildSRPMFromSCMTask(BaseTaskHandler):
             if re.match("%%define\s+%s\s+" % tag, spec, re.M):
                 raise koji.BuildError, "%s is not allowed to be defined in spec file" % tag
 
-    def handler(self, url, build_tag, opts=None):
+    def handler(self, url, build_tag, repo_id):
         global options
 
         # will throw a BuildError if the url is invalid
         scm = SCM(url)
         scm.assert_allowed(options.allowed_scms)
 
+        repo_info = session.repoInfo(repo_id, strict=True)
+        event_id = repo_info['create_event']
+        build_tag = session.getTag(build_tag, strict=True, event=event_id)
+
         # need DNS in the chroot because "make srpm" may need to contact
         # a SCM or lookaside cache to retrieve the srpm contents
         rootopts = {'install_group': 'srpm-build',
-                    'setup_dns': True}
-        if opts.get('repo_id') is not None:
-            rootopts['repo_id'] = opts['repo_id']
+                    'setup_dns': True,
+                    'repo_id': repo_info['id']}
         br_arch = self.find_arch('noarch', session.host.getHost(), session.getBuildConfig(build_tag['id']))
         broot = BuildRoot(build_tag['id'], br_arch, self.id, **rootopts)
 
@@ -2392,9 +2394,13 @@ class NewRepoTask(BaseTaskHandler):
         #see if we can find a previous repo to update from
         oldrepo = session.getRepo(tinfo['id'], state=koji.REPO_READY)
         subtasks = {}
+        external_repos = session.getExternalRepoList(tinfo['id'], event=event)
         for arch in arches:
+            arglist = [repo_id, arch, oldrepo]
+            if external_repos:
+                arglist.append(external_repos)
             subtasks[arch] = session.host.subtask(method='createrepo',
-                                                  arglist=[repo_id, arch, oldrepo],
+                                                  arglist=arglist,
                                                   label=arch,
                                                   parent=self.id,
                                                   arch='noarch')
@@ -2413,62 +2419,100 @@ class NewRepoTask(BaseTaskHandler):
 class CreaterepoTask(BaseTaskHandler):
 
     Methods = ['createrepo']
-    #XXX - set weight?
     _taskWeight = 0.5
 
-    def handler(self, repo_id, arch, oldrepo):
+    def handler(self, repo_id, arch, oldrepo, external_repos=None):
         #arch is the arch of the repo, not the task
-        rinfo = session.repoInfo(repo_id)
+        rinfo = session.repoInfo(repo_id, strict=True)
         if rinfo['state'] != koji.REPO_INIT:
             raise koji.GenericError, "Repo %(id)s not in INIT state (got %(state)s)" % rinfo
-        pathinfo = koji.PathInfo(options.topdir)
-        toprepodir = pathinfo.repo(repo_id, rinfo['tag_name'])
-        repodir = "%s/%s" % (toprepodir, arch)
+        self.pathinfo = koji.PathInfo(options.topdir)
+        toprepodir = self.pathinfo.repo(repo_id, rinfo['tag_name'])
+        self.repodir = '%s/%s' % (toprepodir, arch)
+        if not os.path.isdir(self.repodir):
+            raise koji.GenericError, "Repo directory missing: %s" % self.repodir
         groupdata = os.path.join(toprepodir, 'groups', 'comps.xml')
-        if not os.path.isdir(repodir):
-            raise koji.GenericError, "Repo directory missing: %s" % repodir
         #set up our output dir
-        outdir = "%s/repo" % self.workdir
-        datadir = "%s/repodata" % outdir
-        koji.ensuredir(outdir)
-        pkglist = os.path.join(repodir, 'pkglist')
-        pkgdir = os.path.join(pathinfo.topdir, 'packages/')
-        cmd = ['/usr/bin/createrepo', '-vd', '--outputdir', outdir, '-i', pkglist, '-u', options.pkgurl]
+        self.outdir = '%s/repo' % self.workdir
+        self.datadir = '%s/repodata' % self.outdir
+        pkglist = os.path.join(self.repodir, 'pkglist')
+        if os.path.getsize(pkglist) > 0:
+            # don't call createrepo with an empty pkglist or it'll
+            # add every Koji-managed rpm to the repodata
+            self.create_local_repo(rinfo, arch, pkglist, groupdata, oldrepo)
+
+        external_repos = session.getExternalRepoList(rinfo['tag_id'], event=rinfo['create_event'])
+        if external_repos:
+            self.merge_repos(external_repos, arch, groupdata)
+
+        uploadpath = self.getUploadDir()
+        files = []
+        for f in os.listdir(self.datadir):
+            files.append(f)
+            session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f)
+
+        return [uploadpath, files]
+
+    def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):
+        koji.ensuredir(self.outdir)
+        cmd = ['/usr/bin/createrepo', '-vd', '-o', self.outdir, '-i', pkglist, '-u', options.pkgurl]
         if os.path.isfile(groupdata):
             cmd.extend(['-g', groupdata])
         #attempt to recycle repodata from last repo
         if oldrepo:
-            oldpath = pathinfo.repo(oldrepo['id'], rinfo['tag_name'])
-            olddatadir = "%s/%s/repodata" % (oldpath, arch)
+            oldpath = self.pathinfo.repo(oldrepo['id'], rinfo['tag_name'])
+            olddatadir = '%s/%s/repodata' % (oldpath, arch)
             if not os.path.isdir(olddatadir):
                 self.logger.warn("old repodata is missing: %s" % olddatadir)
             else:
-                koji.ensuredir(datadir)
-                os.system('cp -a %s/* %s' % (olddatadir, datadir))
+                shutil.copytree(olddatadir, self.datadir)
+                oldorigins = os.path.join(self.datadir, 'pkgorigins.gz')
+                if os.path.isfile(oldorigins):
+                    # remove any previous origins file and rely on mergerepos
+                    # to rewrite it (if we have external repos to merge)
+                    os.unlink(oldorigins)
                 cmd.append('--update')
                 if options.createrepo_skip_stat:
                     cmd.append('--skip-stat')
         # note: we can't easily use a cachedir because we do not have write
         # permission. The good news is that with --update we won't need to
         # be scanning many rpms.
+        pkgdir = os.path.join(self.pathinfo.topdir, 'packages/')
         cmd.append(pkgdir)
 
-        logfile = "%s/createrepo.log" % self.workdir
-        uploadpath = self.getUploadDir()
-        #log_output(path, args, outfile, uploadpath, cwd=None, logerror=0, append=0, chroot=None):
-        status = log_output(cmd[0], cmd, logfile, uploadpath, logerror=True)
+        logfile = '%s/createrepo.log' % self.workdir
+        status = log_output(cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
         if not _isSuccess(status):
-            raise koji.GenericError, "failed to create repo: %s" \
+            raise koji.GenericError, 'failed to create repo: %s' \
                     % _parseStatus(status, ' '.join(cmd))
 
-        files = []
-        for f in os.listdir(datadir):
-            if f.endswith('.xml') or f.endswith('.xml.gz') or f.endswith('.sqlite.bz2'):
-                files.append(f)
-                session.uploadWrapper("%s/%s" % (datadir, f), uploadpath, f)
+    def merge_repos(self, external_repos, arch, groupdata):
+        repos = []
+        if os.path.isdir(self.datadir):
+            localdir = '%s/localrepo' % self.workdir
+            os.rename(self.outdir, localdir)
+            koji.ensuredir(self.outdir)
+            repos.append('file://' + localdir + '/')
+
+        for repo in external_repos:
+            ext_url = repo['url']
+            # substitute $arch in the url with the arch of the repo we're generating
+            ext_url = ext_url.replace('$arch', arch)
+            repos.append(ext_url)
+
+        blocklist = self.repodir + '/blocklist'
+        cmd = ['/usr/libexec/kojid/mergerepos', '-a', arch, '-b', blocklist, '-o', self.outdir]
+        if os.path.isfile(groupdata):
+            cmd.extend(['-g', groupdata])
+        for repo in repos:
+            cmd.extend(['-r', repo])
+
+        logfile = '%s/mergerepo.log' % self.workdir
+        status = log_output(cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
+        if not _isSuccess(status):
+            raise koji.GenericError, 'failed to merge repos: %s' \
+                % _parseStatus(status, ' '.join(cmd))
 
-        return [uploadpath, files]
-        
 class WaitrepoTask(BaseTaskHandler):
 
     Methods = ['waitrepo']

builder/mergerepos

@@ -0,0 +1,199 @@
+#!/usr/bin/python
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+# Copyright 2009 Red Hat, Inc.
+# Written by Mike Bonnet <mikeb@redhat.com>
+
+# Merge repos using rules specific to Koji
+# Largely borrowed from the mergerepo script included in createrepo and
+# written by Seth Vidal
+
+import createrepo
+import os.path
+import rpmUtils.miscutils
+import shutil
+import sys
+import tempfile
+import yum
+from optparse import OptionParser
+
+def parse_args(args):
+    """Parse our opts/args"""
+    usage = """
+    mergerepos: take 2 or more repositories and merge their metadata into a new repo using Koji semantics
+              
+    mergerepos --repo=url --repo=url --outputdir=/some/path"""
+    
+    parser = OptionParser(version = "mergerepos 0.1", usage=usage)
+    # query options
+    parser.add_option("-r", "--repo", dest="repos", default=[], action="append",
+                      help="repo url")
+    parser.add_option("-g", "--groupfile", default=None,
+                      help="path to groupfile to include in metadata")
+    parser.add_option("-a", "--arch", dest="arches", default=[], action="append",
+                      help="List of arches to include in the repo")
+    parser.add_option("-b", "--blocked", default=None,
+                      help="A file containing a list of srpm names to exclude from the merged repo")
+    parser.add_option("-o", "--outputdir", default=None, 
+                      help="Location to create the repository")
+    (opts, argsleft) = parser.parse_args(args)
+
+    if len(opts.repos) < 1:
+        parser.print_usage()
+        sys.exit(1)
+
+    # always include noarch
+    if not 'noarch' in opts.arches:
+        opts.arches.append('noarch')
+
+    if not opts.outputdir:
+        parser.error('You must specify an outputdir with -o')
+        sys.exit(1)
+
+    return opts
+
+class RepoMerge():
+    def __init__(self, repolist, arches, groupfile, blocked, outputdir):
+        self.repolist = repolist
+        self.outputdir = outputdir
+        self.mdconf = createrepo.MetaDataConfig()
+        self.mdconf.database = True
+        self.mdconf.verbose = True
+        self.mdconf.changelog_limit = 3
+        self.yumbase = yum.YumBase()
+        self.yumbase._getConfig('/dev/null', init_plugins=False, debuglevel=2)
+        self.yumbase.conf.cachedir = tempfile.mkdtemp()
+        self.yumbase.conf.cache = 0
+        self.archlist = arches
+        self.mdconf.groupfile = groupfile
+        self.blocked = blocked
+
+    def close(self):
+        if self.yumbase is not None:
+            cachedir = self.yumbase.conf.cachedir
+            self.yumbase.close()
+            self.yumbase = None
+            self.mdconf = None
+            if os.path.isdir(cachedir):
+                shutil.rmtree(cachedir)
+
+    def __del__(self):
+        self.close()
+
+    def merge_repos(self):
+        self.yumbase.repos.disableRepo('*')
+        # add our repos and give them a merge rank in the order they appear in
+        # in the repolist
+        count = 0
+        for r in self.repolist:
+            count +=1
+            rid = 'repo%s' % count
+            print >> sys.stderr, 'Adding repo: ' + r
+            n = self.yumbase.add_enable_repo(rid, baseurls=[r])
+            n._merge_rank = count
+
+        #setup our sacks
+        self.yumbase._getSacks(archlist=self.archlist)
+
+        self.sort_and_filter()
+
+    def sort_and_filter(self):
+        """
+        For each package object, check if the srpm name has ever been seen before.
+        If is has not, keep the package.  If it has, check if the srpm name was first seen
+        in the same repo as the current package.  If so, keep the package (it's probably a subpackage
+        of the same build).  If not, delete the package from the package sack.
+        Note that this does allow an external repo to contain multiple versions of the same package,
+        and they will all end up in the repo, but the yum client will ensure that only the latest is
+        installed.
+
+        If the srpm name appears in the blocked package list, any packages generated from the srpm
+        will be deleted from the package sack as well.
+
+        This method will also generate a file called "pkgorigins" and add it to the repo metadata.  This
+        is a tab-separated map of package E:N-V-R.A to repo URL (as specified on the command-line).  This
+        allows a package to be tracked back to its origin, even if the location field in the repodata does
+        not match the original repo location.
+        """
+        # sort the repos by _merge_rank
+        # lowest number is the highest rank (1st place, 2nd place, etc.)
+        repos = self.yumbase.repos.listEnabled()
+        repos.sort(key=lambda o: o._merge_rank)
+
+        seen_srpms = {}
+
+        pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins')
+        origins = file(pkgorigins, 'w')
+
+        for repo in repos:
+            for pkg in repo.sack:
+                srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm)
+                if seen_srpms.has_key(srpm_name):
+                    # We've seen a package created from this srpm before.
+                    # Check if that package was also from this repo, in
+                    # which case it's likely just another subpackage from
+                    # the same build.
+                    if seen_srpms[srpm_name] == pkg.repoid:
+                        origins.write('%s\t%s\n' % (pkg, repo.urls[0]))
+                    else:
+                        repo.sack.delPackage(pkg)
+                elif self.blocked.has_key(srpm_name):
+                    print >> sys.stderr, 'Removing blocked package: %s' % pkg
+                    repo.sack.delPackage(pkg)
+                else:
+                    seen_srpms[srpm_name] = pkg.repoid
+                    origins.write('%s\t%s\n' % (pkg, repo.urls[0]))
+
+        origins.close()
+        self.mdconf.additional_metadata['origin'] = pkgorigins
+
+    def write_metadata(self):
+        self.mdconf.pkglist = self.yumbase.pkgSack
+        self.mdconf.directory = self.outputdir
+        # clean out what was there
+        if os.path.exists(self.mdconf.directory + '/repodata'):
+            shutil.rmtree(self.mdconf.directory + '/repodata')
+
+        if not os.path.exists(self.mdconf.directory):
+            os.makedirs(self.mdconf.directory)
+
+        mdgen = createrepo.MetaDataGenerator(config_obj=self.mdconf)
+        mdgen.doPkgMetadata()
+        mdgen.doRepoMetadata()
+        mdgen.doFinalMove()
+
+def main(args):
+    """main"""
+    opts = parse_args(args)
+
+    if opts.blocked:
+        blocked_fo = file(opts.blocked)
+        blocked_list = blocked_fo.readlines()
+        blocked_fo.close()
+        blocked = dict([(b.strip(), 1) for b in blocked_list])
+    else:
+        blocked = {}
+
+    merge = RepoMerge(opts.repos, opts.arches, opts.groupfile, blocked, opts.outputdir)
+
+    try:
+        merge.merge_repos()
+        merge.write_metadata()
+    finally:
+        merge.close()
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

hub/kojihub.py

@@ -1722,9 +1722,10 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
     q = """INSERT INTO repo(id, create_event, tag_id, state)
     VALUES(%(repo_id)s, %(event_id)s, %(tag_id)s, %(state)s)"""
     _dml(q,locals())
-    # no need to pass explicit event, since this is all one transaction
-    rpms, builds = readTaggedRPMS(tag_id, event=None, inherit=True, latest=True)
-    groups = readTagGroups(tag_id, event=None, inherit=True)
+    rpms, builds = readTaggedRPMS(tag_id, event=event_id, inherit=True, latest=True)
+    groups = readTagGroups(tag_id, event=event_id, inherit=True)
+    blocks = [pkg for pkg in readPackageList(tag_id, event=event_id, inherit=True).values() \
+                  if pkg['blocked']]
     repodir = koji.pathinfo.repo(repo_id, tinfo['name'])
     os.makedirs(repodir)  #should not already exist
     #index builds
@@ -1784,6 +1785,32 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
             for rpminfo in packages.get('src',[]):
                 pkglist.write(rpminfo['path'].split(os.path.join(koji.pathinfo.topdir, 'packages/'))[1] + '\n')
         pkglist.close()
+        #write list of blocked packages
+        blocklist = file(os.path.join(repodir, arch, 'blocklist'), 'w')
+        logger.info("Creating blocked list for %s" % arch)
+        for pkg in blocks:
+            blocklist.write(pkg['package_name'])
+            blocklist.write('\n')
+        blocklist.close()
+
+    # if using an external repo, make sure we've created a directory and pkglist for
+    # every arch in the taglist, or any packages of that arch in the external repo
+    # won't be processed
+    if get_external_repo_list(tinfo['id'], event=event_id):
+        for arch in repo_arches:
+            pkglist = os.path.join(repodir, arch, 'pkglist')
+            if not os.path.exists(pkglist):
+                logger.info("Creating missing package list for %s" % arch)
+                koji.ensuredir(os.path.dirname(pkglist))
+                pkglist_fo = file(pkglist, 'w')
+                pkglist_fo.close()
+                blocklist = file(os.path.join(repodir, arch, 'blocklist'), 'w')
+                logger.info("Creating missing blocked list for %s" % arch)
+                for pkg in blocks:
+                    blocklist.write(pkg['package_name'])
+                    blocklist.write('\n')
+                blocklist.close()
+
     return [repo_id, event_id]
 
 def repo_set_state(repo_id, state, check=True):

koji.spec

@@ -135,6 +135,8 @@ rm -rf $RPM_BUILD_ROOT
 %files builder
 %defattr(-,root,root)
 %{_sbindir}/kojid
+%dir %{_libexecdir}/kojid
+%{_libexecdir}/kojid/mergerepos
 %{_initrddir}/kojid
 %config(noreplace) %{_sysconfdir}/sysconfig/kojid
 %dir %{_sysconfdir}/kojid

www/kojiweb/index.py

@@ -557,6 +557,10 @@ def taskinfo(req, taskID):
     if task['method'] == 'buildArch':
         buildTag = server.getTag(params[1])
         values['buildTag'] = buildTag
+    elif task['method'] == 'buildSRPMFromSCM':
+        if len(params) > 1:
+            buildTag = server.getTag(params[1])
+            values['buildTag'] = buildTag
     elif task['method'] == 'tagBuild':
         destTag = server.getTag(params[0])
         build = server.getBuild(params[1])

www/kojiweb/taskinfo.chtml

@@ -68,10 +68,10 @@
         #if $task.method == 'buildSRPMFromSCM'
         <strong>SCM URL:</strong> $params[0]<br/>
         #if $len($params) > 1
-        <strong>Build Tag:</strong>: <a href="taginfo?tagID=$params[1].name">$params[1].name</a><br/>
+        <strong>Build Tag:</strong>: <a href="taginfo?tagID=$buildTag.name">$buildTag.name</a><br/>
         #end if
         #if $len($params) > 2
-        $printOpts($params[2])
+        <strong>Repo ID:</strong>: $params[2]<br/>
         #end if
         #elif $task.method == 'buildSRPMFromCVS'
         <strong>CVS URL:</strong> $params[0]
@@ -129,7 +129,10 @@
         #set $oldrepo = $params[2]
         #if $oldrepo
         <strong>Old Repo ID:</strong> $oldrepo.id<br/>
-        <strong>Old Repo Creation:</strong> $koji.formatTimeLong($oldrepo.creation_time)
+        <strong>Old Repo Creation:</strong> $koji.formatTimeLong($oldrepo.creation_time)<br/>
+        #end if
+        #if $len($params) > 3
+        <strong>External Repos:</strong> $printValue(None, [ext['external_repo_name'] for ext in $params[3]])<br/>
         #end if
         #elif $task.method == 'dependantTask'
         <strong>Dependant Tasks:</strong><br/>