refactor uploaded file data
This commit is contained in:
Mike McLean
parent
8e08e66d9a
commit
fb5da35edf
2 changed files with 59 additions and 79 deletions
|
|
@ -5160,9 +5160,9 @@ class NewDistRepoTask(BaseTaskHandler):
|
|||
for arch in arch32s:
|
||||
# move the 32-bit task output to the final resting place
|
||||
# so the 64-bit arches can use it for multilib
|
||||
upload, files, sigmap = results[subtasks[arch]]
|
||||
upload, files = results[subtasks[arch]]
|
||||
self.session.host.distRepoMove(
|
||||
repo_id, upload, files, arch, sigmap)
|
||||
repo_id, upload, files, arch)
|
||||
for arch in canonArches:
|
||||
# do the other arches
|
||||
if arch not in arch32s:
|
||||
|
|
@ -5180,9 +5180,9 @@ class NewDistRepoTask(BaseTaskHandler):
|
|||
# already moved above
|
||||
continue
|
||||
#else
|
||||
upload, files, sigmap = results[subtasks[arch]]
|
||||
upload, files = results[subtasks[arch]]
|
||||
self.session.host.distRepoMove(
|
||||
repo_id, upload, files, arch, sigmap)
|
||||
repo_id, upload, files, arch)
|
||||
self.session.host.repoDone(repo_id, data, expire=False)
|
||||
return 'Dist repository #%s successfully generated' % repo_id
|
||||
|
||||
|
|
@ -5223,10 +5223,8 @@ class createDistRepoTask(BaseTaskHandler):
|
|||
|
||||
# set up our output dir
|
||||
self.repodir = '%s/repo' % self.workdir
|
||||
self.repo_files = []
|
||||
koji.ensuredir(self.repodir)
|
||||
self.outdir = self.repodir # workaround create_local_repo use
|
||||
datadir = '%s/repodata' % self.repodir
|
||||
self.sigmap = {}
|
||||
self.subrepos = set()
|
||||
|
||||
# gather oldpkgs data if delta option in use
|
||||
|
|
@ -5266,27 +5264,36 @@ class createDistRepoTask(BaseTaskHandler):
|
|||
groupdata, oldpkgs=oldpkgs,
|
||||
logname='createrepo_%s' % subrepo)
|
||||
if len(self.kojipkgs) == 0:
|
||||
fo = file(os.path.join(datadir, "EMPTY_REPO"), 'w')
|
||||
fo.write("This repo is empty because its tag has no content for this arch\n")
|
||||
fo.close()
|
||||
fn = os.path.join(self.repodir, "repodata", "EMPTY_REPO")
|
||||
with open(fn, 'w') as fp:
|
||||
fp.write("This repo is empty because its tag has no content "
|
||||
"for this arch\n")
|
||||
|
||||
# upload repo files
|
||||
files = ['pkglist', 'kojipkgs']
|
||||
for subrepo in self.subrepos:
|
||||
files.append('%s/pkglist' % subrepo)
|
||||
for f in os.listdir(datadir):
|
||||
for f in os.listdir('%s/repodata' % self.repodir):
|
||||
# repo metadata files
|
||||
files.append(f)
|
||||
self.session.uploadWrapper('%s/%s' % (datadir, f),
|
||||
self.uploadpath, f)
|
||||
self.upload_repo_file("repodata/%s" % f)
|
||||
if opts['delta']:
|
||||
ddir = os.path.join(self.repodir, 'drpms')
|
||||
for f in os.listdir(ddir):
|
||||
files.append(f)
|
||||
self.session.uploadWrapper('%s/%s' % (ddir, f),
|
||||
self.uploadpath, f)
|
||||
for f in os.listdir('%s/drpms' % self.repodir):
|
||||
self.upload_repo_file("drpms/%s" % f)
|
||||
|
||||
return [self.uploadpath, files, self.sigmap.items()]
|
||||
return [self.uploadpath, self.repo_files]
|
||||
|
||||
def upload_repo_file(self, relpath):
|
||||
"""Upload a file from the repo
|
||||
|
||||
relpath should be relative to self.repodir
|
||||
"""
|
||||
localpath = '%s/%s' % (self.repodir, relpath)
|
||||
reldir = os.path.dirname(relpath)
|
||||
if reldir:
|
||||
uploadpath = "%s/%s" % (self.uploadpath, reldir)
|
||||
fn = os.path.basename(relpath)
|
||||
else:
|
||||
uploadpath = self.uploadpath
|
||||
fn = relpath
|
||||
self.session.uploadWrapper(localpath, uploadpath, fn)
|
||||
self.repo_files.append(relpath)
|
||||
|
||||
def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None, logname=None):
|
||||
"""Run createrepo
|
||||
|
|
@ -5452,8 +5459,6 @@ enabled=1
|
|||
rpminfo['_pkgpath'] = dep_path
|
||||
rpminfo['_multilib'] = True
|
||||
self.kojipkgs[bnp] = rpminfo
|
||||
self.sigmap[rpminfo['id']] = rpminfo['sigkey']
|
||||
|
||||
|
||||
def pick_key(self, keys, avail_keys):
|
||||
best = None
|
||||
|
|
@ -5524,7 +5529,6 @@ enabled=1
|
|||
# we'll raise an error below
|
||||
else:
|
||||
bnp = os.path.basename(pkgpath)
|
||||
self.sigmap[rpminfo['id']] = rpminfo['sigkey']
|
||||
rpminfo['_pkgpath'] = pkgpath
|
||||
kojipkgs[bnp] = rpminfo
|
||||
self.kojipkgs = kojipkgs
|
||||
|
|
@ -5605,12 +5609,9 @@ enabled=1
|
|||
fo.write(line)
|
||||
|
||||
# and upload too
|
||||
self.session.uploadWrapper('%s/pkglist' % self.repodir,
|
||||
self.uploadpath, 'pkglist')
|
||||
self.upload_repo_file('pkglist')
|
||||
for subrepo in subrepo_pkgs:
|
||||
self.session.uploadWrapper(
|
||||
'%s/%s/pkglist' % (self.repodir, subrepo),
|
||||
'%s/%s' % (self.uploadpath, subrepo), 'pkglist')
|
||||
self.upload_repo_file('%s/pkglist' % subrepo)
|
||||
|
||||
def write_kojipkgs(self):
|
||||
filename = os.path.join(self.repodir, 'kojipkgs')
|
||||
|
|
@ -5620,8 +5621,7 @@ enabled=1
|
|||
finally:
|
||||
datafile.close()
|
||||
# and upload too
|
||||
self.session.uploadWrapper(filename, self.uploadpath, 'kojipkgs')
|
||||
|
||||
self.upload_repo_file('kojipkgs')
|
||||
|
||||
|
||||
class WaitrepoTask(BaseTaskHandler):
|
||||
|
|
|
|||
|
|
@ -12633,7 +12633,7 @@ class HostExports(object):
|
|||
koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire)
|
||||
|
||||
|
||||
def distRepoMove(self, repo_id, uploadpath, files, arch, sigmap):
|
||||
def distRepoMove(self, repo_id, uploadpath, files, arch):
|
||||
"""
|
||||
Move one arch of a dist repo into its final location
|
||||
|
||||
|
|
@ -12644,13 +12644,10 @@ class HostExports(object):
|
|||
uploadpath - where the uploaded files are
|
||||
files - a list of the uploaded file names
|
||||
arch - the arch of the repo
|
||||
sigmap - a list of [rpm_id, sig] pairs
|
||||
|
||||
The rpms from sigmap should match the contents of the uploaded pkglist
|
||||
file.
|
||||
|
||||
In sigmap, use sig=None to use the primary copy of the rpm instead of a
|
||||
signed copy.
|
||||
The uploaded files should include:
|
||||
- kojipkgs: json file with information about the component rpms
|
||||
- repo metadata files
|
||||
"""
|
||||
host = Host()
|
||||
host.verify()
|
||||
|
|
@ -12671,43 +12668,28 @@ class HostExports(object):
|
|||
with open(fn) as fp:
|
||||
kojipkgs = json.load(fp)
|
||||
|
||||
# Figure out subrepos
|
||||
subrepos = set()
|
||||
for bnp in kojipkgs:
|
||||
rpminfo = kojipkgs[bnp]
|
||||
subrepo = rpminfo.get('_subrepo')
|
||||
if subrepo:
|
||||
subrepos.add(subrepo)
|
||||
|
||||
# Figure out where to send the uploaded files
|
||||
file_moves = []
|
||||
datadir = "%s/repodata" % archdir
|
||||
for fn in files:
|
||||
src = "%s/%s/%s" % (workdir, uploadpath, fn)
|
||||
if fn.endswith('.drpm'):
|
||||
koji.ensuredir(os.path.join(archdir, 'drpms'))
|
||||
dst = "%s/drpms/%s" % (archdir, fn)
|
||||
elif fn.endswith('pkglist') or fn.endswith('kojipkgs'):
|
||||
dst = '%s/%s' % (archdir, fn)
|
||||
else:
|
||||
dst = "%s/%s" % (datadir, fn)
|
||||
for relpath in files:
|
||||
src = "%s/%s/%s" % (workdir, uploadpath, relpath)
|
||||
dst = "%s/%s" % (archdir, relpath)
|
||||
if not os.path.exists(src):
|
||||
raise koji.GenericError("uploaded file missing: %s" % src)
|
||||
file_moves.append([src, dst])
|
||||
|
||||
dirnames = set([os.path.dirname(fm[1]) for fm in file_moves])
|
||||
for dirname in dirnames:
|
||||
koji.ensuredir(dirname)
|
||||
for src, dst in file_moves:
|
||||
safer_move(src, dst)
|
||||
|
||||
koji.ensuredir(datadir)
|
||||
|
||||
# get rpms
|
||||
build_dirs = {}
|
||||
rpmdata = {}
|
||||
for rpm_id, sigkey in sigmap:
|
||||
rpminfo = get_rpm(rpm_id, strict=True)
|
||||
for bnp in kojipkgs:
|
||||
rpminfo = kojipkgs[bnp]
|
||||
rpm_id = rpminfo['id']
|
||||
sigkey = rpminfo['sigkey']
|
||||
_rpminfo = get_rpm(rpm_id, strict=True)
|
||||
for key in _rpminfo:
|
||||
if key not in rpminfo or rpminfo[key] != _rpminfo[key]:
|
||||
raise koji.GenericError(
|
||||
'kojipkgs entry does not match db: file %s, key %s'
|
||||
% (bnp, key))
|
||||
if sigkey is None or sigkey == '':
|
||||
relpath = koji.pathinfo.rpm(rpminfo)
|
||||
else:
|
||||
|
|
@ -12720,20 +12702,18 @@ class HostExports(object):
|
|||
builddir = koji.pathinfo.build(binfo)
|
||||
build_dirs[rpminfo['build_id']] = builddir
|
||||
rpminfo['_fullpath'] = os.path.join(builddir, relpath)
|
||||
basename = os.path.basename(relpath)
|
||||
rpmdata[basename] = rpminfo
|
||||
rpmdata[bnp] = rpminfo
|
||||
|
||||
# sanity check
|
||||
for fn in rpmdata:
|
||||
if fn not in pkglist and fn not in debuglist:
|
||||
raise koji.GenericError("No signature data for: %s" % fn)
|
||||
for fn in pkglist.union(debuglist):
|
||||
if fn not in rpmdata:
|
||||
raise koji.GenericError("RPM missing from pkglist: %s" % fn)
|
||||
# move the uploaded files
|
||||
dirnames = set([os.path.dirname(fm[1]) for fm in file_moves])
|
||||
for dirname in dirnames:
|
||||
koji.ensuredir(dirname)
|
||||
for src, dst in file_moves:
|
||||
safer_move(src, dst)
|
||||
|
||||
# hardlink or copy the rpms into the final repodir
|
||||
# TODO: properly consider split-volume functionality
|
||||
for fn in rpmdata:
|
||||
# hardlink or copy the rpms into the final repodir
|
||||
# TODO: properly consider split-volume functionality
|
||||
rpminfo = rpmdata[fn]
|
||||
rpmpath = rpminfo['_fullpath']
|
||||
bnp = fn
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue