refactor uploaded file data

builder/kojid

@@ -5160,9 +5160,9 @@ class NewDistRepoTask(BaseTaskHandler):
             for arch in arch32s:
                 # move the 32-bit task output to the final resting place
                 # so the 64-bit arches can use it for multilib
-                upload, files, sigmap = results[subtasks[arch]]
+                upload, files = results[subtasks[arch]]
                 self.session.host.distRepoMove(
-                    repo_id, upload, files, arch, sigmap)
+                    repo_id, upload, files, arch)
         for arch in canonArches:
             # do the other arches
             if arch not in arch32s:
@@ -5180,9 +5180,9 @@ class NewDistRepoTask(BaseTaskHandler):
                 # already moved above
                 continue
             #else
-            upload, files, sigmap = results[subtasks[arch]]
+            upload, files = results[subtasks[arch]]
             self.session.host.distRepoMove(
-                repo_id, upload, files, arch, sigmap)
+                repo_id, upload, files, arch)
         self.session.host.repoDone(repo_id, data, expire=False)
         return 'Dist repository #%s successfully generated' % repo_id
 
@@ -5223,10 +5223,8 @@ class createDistRepoTask(BaseTaskHandler):
 
         # set up our output dir
         self.repodir = '%s/repo' % self.workdir
+        self.repo_files = []
         koji.ensuredir(self.repodir)
-        self.outdir = self.repodir  # workaround create_local_repo use
-        datadir = '%s/repodata' % self.repodir
-        self.sigmap = {}
         self.subrepos = set()
 
         # gather oldpkgs data if delta option in use
@@ -5266,27 +5264,36 @@ class createDistRepoTask(BaseTaskHandler):
                     groupdata, oldpkgs=oldpkgs,
                     logname='createrepo_%s' % subrepo)
         if len(self.kojipkgs) == 0:
-            fo = file(os.path.join(datadir, "EMPTY_REPO"), 'w')
+            fn = os.path.join(self.repodir, "repodata", "EMPTY_REPO")
-            fo.write("This repo is empty because its tag has no content for this arch\n")
+            with open(fn, 'w') as fp:
-            fo.close()
+                fp.write("This repo is empty because its tag has no content "
+                         "for this arch\n")
 
         # upload repo files
-        files = ['pkglist', 'kojipkgs']
+        for f in os.listdir('%s/repodata' % self.repodir):
-        for subrepo in self.subrepos:
-            files.append('%s/pkglist' % subrepo)
-        for f in os.listdir(datadir):
             # repo metadata files
-            files.append(f)
+            self.upload_repo_file("repodata/%s" % f)
-            self.session.uploadWrapper('%s/%s' % (datadir, f),
-                self.uploadpath, f)
         if opts['delta']:
-            ddir = os.path.join(self.repodir, 'drpms')
+            for f in os.listdir('%s/drpms' % self.repodir):
-            for f in os.listdir(ddir):
+                self.upload_repo_file("drpms/%s" % f)
-                files.append(f)
-                self.session.uploadWrapper('%s/%s' % (ddir, f),
-                    self.uploadpath, f)
 
-        return [self.uploadpath, files, self.sigmap.items()]
+        return [self.uploadpath, self.repo_files]
+
+    def upload_repo_file(self, relpath):
+        """Upload a file from the repo
+
+        relpath should be relative to self.repodir
+        """
+        localpath = '%s/%s' % (self.repodir, relpath)
+        reldir = os.path.dirname(relpath)
+        if reldir:
+            uploadpath = "%s/%s" % (self.uploadpath, reldir)
+            fn = os.path.basename(relpath)
+        else:
+            uploadpath = self.uploadpath
+            fn = relpath
+        self.session.uploadWrapper(localpath, uploadpath, fn)
+        self.repo_files.append(relpath)
 
     def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None, logname=None):
         """Run createrepo
@@ -5452,8 +5459,6 @@ enabled=1
             rpminfo['_pkgpath'] = dep_path
             rpminfo['_multilib'] = True
             self.kojipkgs[bnp] = rpminfo
-            self.sigmap[rpminfo['id']] = rpminfo['sigkey']
-
 
     def pick_key(self, keys, avail_keys):
         best = None
@@ -5524,7 +5529,6 @@ enabled=1
                 # we'll raise an error below
             else:
                 bnp = os.path.basename(pkgpath)
-                self.sigmap[rpminfo['id']] = rpminfo['sigkey']
                 rpminfo['_pkgpath'] = pkgpath
                 kojipkgs[bnp] = rpminfo
         self.kojipkgs = kojipkgs
@@ -5605,12 +5609,9 @@ enabled=1
                     fo.write(line)
 
         # and upload too
-        self.session.uploadWrapper('%s/pkglist' % self.repodir,
+        self.upload_repo_file('pkglist')
-                    self.uploadpath, 'pkglist')
         for subrepo in subrepo_pkgs:
-            self.session.uploadWrapper(
+            self.upload_repo_file('%s/pkglist' % subrepo)
-                    '%s/%s/pkglist' % (self.repodir, subrepo),
-                    '%s/%s' % (self.uploadpath, subrepo), 'pkglist')
 
     def write_kojipkgs(self):
         filename = os.path.join(self.repodir, 'kojipkgs')
@@ -5620,8 +5621,7 @@ enabled=1
         finally:
             datafile.close()
         # and upload too
-        self.session.uploadWrapper(filename, self.uploadpath, 'kojipkgs')
+        self.upload_repo_file('kojipkgs')
-
 
 
 class WaitrepoTask(BaseTaskHandler):

hub/kojihub.py

@@ -12633,7 +12633,7 @@ class HostExports(object):
         koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire)
 
 
-    def distRepoMove(self, repo_id, uploadpath, files, arch, sigmap):
+    def distRepoMove(self, repo_id, uploadpath, files, arch):
         """
         Move one arch of a dist repo into its final location
 
@@ -12644,13 +12644,10 @@ class HostExports(object):
         uploadpath - where the uploaded files are
         files - a list of the uploaded file names
         arch - the arch of the repo
-        sigmap - a list of [rpm_id, sig] pairs
 
-        The rpms from sigmap should match the contents of the uploaded pkglist
+        The uploaded files should include:
-        file.
+            - kojipkgs: json file with information about the component rpms
-
+            - repo metadata files
-        In sigmap, use sig=None to use the primary copy of the rpm instead of a
-        signed copy.
         """
         host = Host()
         host.verify()
@@ -12671,43 +12668,28 @@ class HostExports(object):
         with open(fn) as fp:
             kojipkgs = json.load(fp)
 
-        # Figure out subrepos
-        subrepos = set()
-        for bnp in kojipkgs:
-            rpminfo = kojipkgs[bnp]
-            subrepo = rpminfo.get('_subrepo')
-            if subrepo:
-                subrepos.add(subrepo)
-
         # Figure out where to send the uploaded files
         file_moves = []
-        datadir = "%s/repodata" % archdir
+        for relpath in files:
-        for fn in files:
+            src = "%s/%s/%s" % (workdir, uploadpath, relpath)
-            src = "%s/%s/%s" % (workdir, uploadpath, fn)
+            dst = "%s/%s" % (archdir, relpath)
-            if fn.endswith('.drpm'):
-                koji.ensuredir(os.path.join(archdir, 'drpms'))
-                dst = "%s/drpms/%s" % (archdir, fn)
-            elif fn.endswith('pkglist') or fn.endswith('kojipkgs'):
-                dst = '%s/%s' % (archdir, fn)
-            else:
-                dst = "%s/%s" % (datadir, fn)
             if not os.path.exists(src):
                 raise koji.GenericError("uploaded file missing: %s" % src)
             file_moves.append([src, dst])
 
-        dirnames = set([os.path.dirname(fm[1]) for fm in file_moves])
-        for dirname in dirnames:
-            koji.ensuredir(dirname)
-        for src, dst in file_moves:
-            safer_move(src, dst)
-
-        koji.ensuredir(datadir)
-
         # get rpms
         build_dirs = {}
         rpmdata = {}
-        for rpm_id, sigkey in sigmap:
+        for bnp in kojipkgs:
-            rpminfo = get_rpm(rpm_id, strict=True)
+            rpminfo = kojipkgs[bnp]
+            rpm_id = rpminfo['id']
+            sigkey = rpminfo['sigkey']
+            _rpminfo = get_rpm(rpm_id, strict=True)
+            for key in _rpminfo:
+                if key not in rpminfo or rpminfo[key] != _rpminfo[key]:
+                    raise koji.GenericError(
+                            'kojipkgs entry does not match db: file %s, key %s'
+                            % (bnp, key))
             if sigkey is None or sigkey == '':
                 relpath = koji.pathinfo.rpm(rpminfo)
             else:
@@ -12720,20 +12702,18 @@ class HostExports(object):
                 builddir = koji.pathinfo.build(binfo)
                 build_dirs[rpminfo['build_id']] = builddir
             rpminfo['_fullpath'] = os.path.join(builddir, relpath)
-            basename = os.path.basename(relpath)
+            rpmdata[bnp] = rpminfo
-            rpmdata[basename] = rpminfo
 
-        # sanity check
+        # move the uploaded files
-        for fn in rpmdata:
+        dirnames = set([os.path.dirname(fm[1]) for fm in file_moves])
-            if fn not in pkglist and fn not in debuglist:
+        for dirname in dirnames:
-                raise koji.GenericError("No signature data for: %s" % fn)
+            koji.ensuredir(dirname)
-        for fn in pkglist.union(debuglist):
+        for src, dst in file_moves:
-            if fn  not in rpmdata:
+            safer_move(src, dst)
-                raise koji.GenericError("RPM missing from pkglist: %s" % fn)
 
+        # hardlink or copy the rpms into the final repodir
+        # TODO: properly consider split-volume functionality
         for fn in rpmdata:
-            # hardlink or copy the rpms into the final repodir
-            # TODO: properly consider split-volume functionality
             rpminfo = rpmdata[fn]
             rpmpath = rpminfo['_fullpath']
             bnp = fn