PR#1066: Simple mode for mergerepos

Merges #1066
https://pagure.io/koji/pull-request/1066

builder/kojid

@@ -5098,30 +5098,52 @@ class CreaterepoTask(BaseTaskHandler):
                     % parseStatus(status, ' '.join(cmd)))
 
     def merge_repos(self, external_repos, arch, groupdata):
-        repos = []
+        # group repos by merge type
+        repos_by_mode = {}
+        for repo in external_repos:
+            repos_by_mode.setdefault(
+                    repo.get('merge_mode', 'koji'), []).append(repo)
+
+        # figure out merge mode
+        if len(repos_by_mode) > 1:
+            # TODO: eventually support mixing merge modes
+            raise koji.GenericError('Found multiple merge modes for external '
+                    'repos: %s\n' % repos_by_mode.keys())
+        merge_mode = repos_by_mode.keys()[0]
+
+        # move current repo to the premerge location
         localdir = '%s/repo_%s_premerge' % (self.workdir, self.repo_id)
         os.rename(self.outdir, localdir)
         koji.ensuredir(self.outdir)
-        repos.append('file://' + localdir + '/')
 
+        # generate repo url list, starting with our local premerge repo
+        repos = ['file://' + localdir + '/']
         for repo in external_repos:
             ext_url = repo['url']
             # substitute $arch in the url with the arch of the repo we're generating
             ext_url = ext_url.replace('$arch', arch)
             repos.append(ext_url)
 
-        blocklist = self.repodir + '/blocklist'
-        if self.options.use_createrepo_c:
+        # construct command
+        if merge_mode == 'simple':
+            # currently only supported by our own mergerepos script
+            # (we need it to write pkgorigins)
+            cmd = ['/usr/libexec/kojid/mergerepos',
+                   '--mode', 'simple',
+                   '--tempdir', self.workdir]
+        elif self.options.use_createrepo_c:
             cmd = ['/usr/bin/mergerepo_c', '--koji']
         else:
             cmd = ['/usr/libexec/kojid/mergerepos']
             cmd.extend(['--tempdir', self.workdir])
+        blocklist = self.repodir + '/blocklist'
         cmd.extend(['-a', arch, '-b', blocklist, '-o', self.outdir])
         if os.path.isfile(groupdata):
             cmd.extend(['-g', groupdata])
         for repo in repos:
             cmd.extend(['-r', repo])
 
+        # run command
         logfile = '%s/mergerepos.log' % self.workdir
         status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
         if not isSuccess(status):

builder/mergerepos

@@ -72,6 +72,7 @@ def parse_args(args):
                       help="List of arches to include in the repo")
     parser.add_option("-b", "--blocked", default=None,
                       help="A file containing a list of srpm names to exclude from the merged repo")
+    parser.add_option("--mode", default='koji', help="Select the merge mode")
     parser.add_option("-o", "--outputdir", default=None,
                       help="Location to create the repository")
     parser.add_option("--tempdir", default=None,
@@ -112,7 +113,8 @@ def make_const_func(value):
 
 
 class RepoMerge(object):
-    def __init__(self, repolist, arches, groupfile, blocked, outputdir, tempdir=None):
+    def __init__(self, repolist, arches, groupfile, blocked, outputdir,
+                 tempdir=None, mode='koji'):
         self.repolist = repolist
         self.outputdir = outputdir
         self.tempdir = tempdir
@@ -134,6 +136,7 @@ class RepoMerge(object):
         self.archlist = arches
         self.mdconf.groupfile = groupfile
         self.blocked = blocked
+        self.mode = mode
 
     def close(self):
         if self.yumbase is not None:
@@ -182,6 +185,10 @@ class RepoMerge(object):
         allows a package to be tracked back to its origin, even if the location field in the repodata does
         not match the original repo location.
         """
+
+        if self.mode == 'simple':
+            return self.do_simple_sort()
+
         # sort the repos by _merge_rank
         # lowest number is the highest rank (1st place, 2nd place, etc.)
         repos = self.yumbase.repos.listEnabled()
@@ -264,6 +271,55 @@ class RepoMerge(object):
         origins.close()
         self.mdconf.additional_metadata['origin'] = pkgorigins
 
+    def do_simple_sort(self):
+        """
+        Handle the 'sort_and_filter' case when mode=simple
+
+        As the name implies, this is a much simpler approach. Mainly, we need
+        to generate the pkgorigins file.
+        """
+
+        # sort the repos by _merge_rank
+        # lowest number is the highest rank (1st place, 2nd place, etc.)
+        repos = self.yumbase.repos.listEnabled()
+        repos.sort(key=lambda o: o._merge_rank)
+
+        # TODO: reduce duplication between this function and sort_and_filter()
+
+        # We lack the complex filtration of mode=koji, but we still need to:
+        # - fix urls for primary repo
+        # - enforce blocked list
+        for reponum, repo in enumerate(repos):
+            for pkg in repo.sack:
+                if reponum == 0 and not pkg.basepath:
+                    # this is the first repo (i.e. the koji repo) and appears
+                    # to be using relative urls
+                    #XXX - kind of a hack, but yum leaves us little choice
+                    #force the pkg object to report a relative location
+                    loc = """<location href="%s"/>\n""" % yum.misc.to_xml(pkg.remote_path, attrib=True)
+                    pkg._return_remote_location = make_const_func(loc)
+
+        pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins')
+        origins = open(pkgorigins, 'w')
+
+        seen_rpms = {}
+        for repo in repos:
+            for pkg in repo.sack:
+                srpm_name, ver, rel, epoch, arch = rpmUtils.miscutils.splitFilename(pkg.sourcerpm)
+                pkg_nvra = str(pkg)
+                if pkg_nvra in seen_rpms:
+                    sys.stderr.write('Duplicate rpm: %s\n' % pkg_nvra)
+                    # note: we warn, but do not omit it
+                if srpm_name in self.blocked:
+                    sys.stderr.write('Ignoring blocked package: %s\n\n' %
+                                     pkg.sourcerpm)
+                    repo.sack.delPackage(pkg)
+                if pkg_nvra not in seen_rpms:
+                    origins.write('%s\t%s\n' % (pkg_nvra, repo.urls[0]))
+                seen_rpms[pkg_nvra] = 1
+        origins.close()
+        self.mdconf.additional_metadata['origin'] = pkgorigins
+
     def write_metadata(self):
         self.mdconf.pkglist = self.yumbase.pkgSack
         self.mdconf.directory = self.outputdir
@@ -292,7 +348,7 @@ def main(args):
         blocked = {}
 
     merge = RepoMerge(opts.repos, opts.arches, opts.groupfile, blocked,
-                      opts.outputdir, opts.tempdir)
+                      opts.outputdir, opts.tempdir, opts.mode)
 
     try:
         merge.merge_repos()

cli/koji_cli/commands.py

@@ -5177,17 +5177,19 @@ def anon_handle_list_external_repos(goptions, session, args):
         header1 = "%-25s %s" % ("External repo name", "URL")
         header2 = "%s %s" % ("-"*25, "-"*40)
     elif format == "tag":
-        format = "%(priority)-3i %(external_repo_name)-25s %(url)s"
-        header1 = "%-3s %-25s %s" % ("Pri", "External repo name", "URL")
-        header2 = "%s %s %s" % ("-"*3, "-"*25, "-"*40)
+        format = "%(priority)-3i %(external_repo_name)-25s %(merge_mode)-10s %(url)s"
+        header1 = "%-3s %-25s %-10s URL" % ("Pri", "External repo name", "Mode")
+        header2 = "%s %s %s %s" % ("-"*3, "-"*25, "-"*10, "-"*40)
     elif format == "multitag":
-        format = "%(tag_name)-20s %(priority)-3i %(external_repo_name)s"
-        header1 = "%-20s %-3s %s" % ("Tag", "Pri", "External repo name")
-        header2 = "%s %s %s" % ("-"*20, "-"*3, "-"*25)
+        format = "%(tag_name)-20s %(priority)-3i %(merge_mode)-10s %(external_repo_name)s"
+        header1 = "%-20s %-3s %-10s %s" % ("Tag", "Pri", "Mode", "External repo name")
+        header2 = "%s %s %s %s" % ("-"*20, "-"*3, "-"*10, "-"*25)
     if not options.quiet:
         print(header1)
         print(header2)
     for rinfo in data:
+        # older hubs do not support merge_mode
+        rinfo.setdefault('merge_mode', None)
         print(format % rinfo)
 
 
@@ -5227,8 +5229,14 @@ def handle_add_external_repo(goptions, session, args):
                       help=_("Also add repo to tag. Use tag::N to set priority"))
     parser.add_option("-p", "--priority", type='int',
                       help=_("Set priority (when adding to tag)"))
+    parser.add_option("-m", "--mode", help=_("Set merge mode"))
     (options, args) = parser.parse_args(args)
     activate_session(session, goptions)
+    if options.mode:
+        if options.mode not in koji.REPO_MERGE_MODES:
+            parser.error('Invalid mode: %s' % options.mode)
+        if not options.tag:
+            parser.error('The --mode option can only be used with --tag')
     if len(args) == 1:
         name = args[0]
         rinfo = session.getExternalRepo(name, strict=True)
@@ -5249,7 +5257,10 @@ def handle_add_external_repo(goptions, session, args):
                     priority = options.priority
                 else:
                     priority = _pick_external_repo_priority(session, tag)
-            session.addExternalRepoToTag(tag, rinfo['name'], priority)
+            callopts = {}
+            if options.mode:
+                callopts['merge_mode'] = options.mode
+            session.addExternalRepoToTag(tag, rinfo['name'], priority, **callopts)
             print("Added external repo %s to tag %s (priority %i)" \
                     % (rinfo['name'], tag, priority))
 
@@ -5261,6 +5272,7 @@ def handle_edit_external_repo(goptions, session, args):
     parser = OptionParser(usage=usage)
     parser.add_option("--url",  help=_("Change the url"))
     parser.add_option("--name",  help=_("Change the name"))
+    parser.add_option("-m", "--mode", help=_("Set merge mode"))
     (options, args) = parser.parse_args(args)
     if len(args) != 1:
         parser.error(_("Incorrect number of arguments"))
@@ -5271,6 +5283,10 @@ def handle_edit_external_repo(goptions, session, args):
         opts['url'] = options.url
     if options.name:
         opts['name'] = options.name
+    if options.mode:
+        if options.mode not in koji.REPO_MERGE_MODES:
+            parser.error('Invalid mode: %s' % options.mode)
+        opts['merge_mode'] = options.mode
     if not opts:
         parser.error(_("No changes specified"))
     activate_session(session, goptions)

docs/schema-upgrade-1.16-1.17.sql

@@ -7,4 +7,7 @@ BEGIN;
 -- Change VARCHAR field for build_target names to TEXT to allow longer names
 ALTER TABLE build_target ALTER COLUMN name TYPE TEXT;
 
+-- Allow different merge modes for mergerepo
+ALTER TABLE tag_external_repos ADD COLUMN merge_mode TEXT DEFAULT 'koji';
+
 COMMIT;

docs/schema.sql

@@ -467,6 +467,7 @@ create table tag_external_repos (
 	tag_id INTEGER NOT NULL REFERENCES tag(id),
 	external_repo_id INTEGER NOT NULL REFERENCES external_repo(id),
 	priority INTEGER NOT NULL,
+	merge_mode TEXT DEFAULT 'koji',
 -- versioned - see earlier description of versioning
 	create_event INTEGER NOT NULL REFERENCES events(id) DEFAULT get_event(),
 	revoke_event INTEGER REFERENCES events(id),

hub/kojihub.py

@@ -3362,11 +3362,14 @@ def delete_external_repo(info):
     update.make_revoke()
     update.execute()
 
-def add_external_repo_to_tag(tag_info, repo_info, priority):
+def add_external_repo_to_tag(tag_info, repo_info, priority, merge_mode='koji'):
     """Add an external repo to a tag"""
 
     context.session.assertPerm('admin')
 
+    if merge_mode not in koji.REPO_MERGE_MODES:
+        raise koji.GenericError('Invalid merge mode: %s' % merge_mode)
+
     tag = get_tag(tag_info, strict=True)
     tag_id = tag['id']
     repo = get_external_repo(repo_info, strict=True)
@@ -3381,7 +3384,8 @@ def add_external_repo_to_tag(tag_info, repo_info, priority):
             (tag['name'], priority))
 
     insert = InsertProcessor('tag_external_repos')
-    insert.set(tag_id=tag_id, external_repo_id=repo_id, priority=priority)
+    insert.set(tag_id=tag_id, external_repo_id=repo_id, priority=priority,
+               merge_mode=merge_mode)
     insert.make_create()
     insert.execute()
 
@@ -3435,14 +3439,23 @@ def get_tag_external_repos(tag_info=None, repo_info=None, event=None):
     external_repo_id
     external_repo_name
     url
+    merge_mode
     priority
     """
     tables = ['tag_external_repos']
     joins = ['tag ON tag_external_repos.tag_id = tag.id',
              'external_repo ON tag_external_repos.external_repo_id = external_repo.id',
              'external_repo_config ON external_repo.id = external_repo_config.external_repo_id']
-    columns = ['tag.id', 'tag.name', 'external_repo.id', 'external_repo.name', 'url', 'priority']
-    aliases = ['tag_id', 'tag_name', 'external_repo_id', 'external_repo_name', 'url', 'priority']
+    fields = {
+            'external_repo.id': 'external_repo_id',
+            'external_repo.name': 'external_repo_name',
+            'priority': 'priority',
+            'tag.id': 'tag_id',
+            'tag.name': 'tag_name',
+            'url': 'url',
+            'merge_mode': 'merge_mode',
+            }
+    columns, aliases = zip(*fields.items())
 
     clauses = [eventCondition(event, table='tag_external_repos'), eventCondition(event, table='external_repo_config')]
     if tag_info:
@@ -3474,6 +3487,7 @@ def get_external_repo_list(tag_info, event=None):
     external_repo_id
     external_repo_name
     url
+    merge_mode
     priority
     """
     tag = get_tag(tag_info, strict=True, event=event)
@@ -6725,7 +6739,7 @@ def query_history(tables=None, **kwargs):
         'external_repo_config': ['external_repo_id', 'url'],
         'host_config': ['host_id', 'arches', 'capacity', 'description', 'comment', 'enabled'],
         'host_channels': ['host_id', 'channel_id'],
-        'tag_external_repos': ['tag_id', 'external_repo_id', 'priority'],
+        'tag_external_repos': ['tag_id', 'external_repo_id', 'priority', 'merge_mode'],
         'tag_listing': ['build_id', 'tag_id'],
         'tag_packages': ['package_id', 'tag_id', 'owner', 'blocked', 'extra_arches'],
         'group_config': ['group_id', 'tag_id', 'blocked', 'exported', 'display_name', 'is_default', 'uservisible',
@@ -9175,10 +9189,11 @@ class RootExports(object):
     editExternalRepo = staticmethod(edit_external_repo)
     deleteExternalRepo = staticmethod(delete_external_repo)
 
-    def addExternalRepoToTag(self, tag_info, repo_info, priority):
+    def addExternalRepoToTag(self, tag_info, repo_info, priority,
+                merge_mode='koji'):
         """Add an external repo to a tag"""
         # wrap the local method so we don't expose the event parameter
-        add_external_repo_to_tag(tag_info, repo_info, priority)
+        add_external_repo_to_tag(tag_info, repo_info, priority, merge_mode)
 
     def removeExternalRepoFromTag(self, tag_info, repo_info):
         """Remove an external repo from a tag"""

koji/__init__.py

@@ -236,6 +236,8 @@ REPO_EXPIRED = REPO_STATES['EXPIRED']
 REPO_DELETED = REPO_STATES['DELETED']
 REPO_PROBLEM = REPO_STATES['PROBLEM']
 
+REPO_MERGE_MODES = set(['koji', 'simple'])
+
 # buildroot states
 BR_STATES = Enum((
     'INIT',