rework signedRepoMove api a bit
This commit is contained in:
Mike McLean
parent
1bfa815b16
commit
cb6a425d7f
3 changed files with 128 additions and 38 deletions
|
|
@ -30,6 +30,10 @@ import koji.plugin
|
|||
import koji.util
|
||||
import koji.tasks
|
||||
import glob
|
||||
try:
|
||||
import json
|
||||
except ImportError: # pragma: no cover
|
||||
import simplejson as json
|
||||
import logging
|
||||
import logging.handlers
|
||||
from koji.daemon import incremental_upload, log_output, TaskManager, SCM
|
||||
|
|
@ -4963,9 +4967,9 @@ class NewSignedRepoTask(BaseTaskHandler):
|
|||
for arch in arch32s:
|
||||
# move the 32-bit task output to the final resting place
|
||||
# so the 64-bit arches can use it for multilib
|
||||
upload, files, keypaths = results[subtasks[arch]]
|
||||
upload, files, sigmap = results[subtasks[arch]]
|
||||
self.session.host.signedRepoMove(
|
||||
repo_id, upload, files, arch, keypaths)
|
||||
repo_id, upload, files, arch, sigmap)
|
||||
for arch in canonArches:
|
||||
# do the other arches
|
||||
if arch not in arch32s:
|
||||
|
|
@ -4983,9 +4987,9 @@ class NewSignedRepoTask(BaseTaskHandler):
|
|||
# already moved above
|
||||
continue
|
||||
#else
|
||||
upload, files, keypaths = results[subtasks[arch]]
|
||||
upload, files, sigmap = results[subtasks[arch]]
|
||||
self.session.host.signedRepoMove(
|
||||
repo_id, upload, files, arch, keypaths)
|
||||
repo_id, upload, files, arch, sigmap)
|
||||
self.session.host.repoDone(repo_id, data, expire=False)
|
||||
return 'Signed repository #%s successfully generated' % repo_id
|
||||
|
||||
|
|
@ -5030,8 +5034,9 @@ class createSignedRepoTask(CreaterepoTask):
|
|||
koji.ensuredir(self.repodir)
|
||||
self.outdir = self.repodir # workaround create_local_repo use
|
||||
self.datadir = '%s/repodata' % self.repodir
|
||||
self.keypaths = {}
|
||||
self.sigmap = {}
|
||||
if len(opts['delta']) > 0:
|
||||
# XXX raw path in options
|
||||
for path in opts['delta']:
|
||||
if not os.path.exists(path):
|
||||
raise koji.GenericError(
|
||||
|
|
@ -5040,6 +5045,7 @@ class createSignedRepoTask(CreaterepoTask):
|
|||
self.pkglist = self.make_pkglist(tag, arch, keys, opts)
|
||||
if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch):
|
||||
self.do_multilib(arch, self.archmap[arch], opts['multilib'])
|
||||
self.write_kojipkgs()
|
||||
self.logger.debug('package list is %s' % self.pkglist)
|
||||
self.session.uploadWrapper(self.pkglist, self.uploadpath,
|
||||
os.path.basename(self.pkglist))
|
||||
|
|
@ -5066,7 +5072,7 @@ class createSignedRepoTask(CreaterepoTask):
|
|||
files.append(f)
|
||||
self.session.uploadWrapper('%s/%s' % (ddir, f),
|
||||
self.uploadpath, f)
|
||||
return [self.uploadpath, files, self.keypaths]
|
||||
return [self.uploadpath, files, self.sigmap]
|
||||
|
||||
def do_multilib(self, arch, ml_arch, conf):
|
||||
self.repo_id = self.rinfo['id']
|
||||
|
|
@ -5076,7 +5082,7 @@ class createSignedRepoTask(CreaterepoTask):
|
|||
ml_true = set() # multilib packages we need to include before depsolve
|
||||
ml_conf = os.path.join(self.pathinfo.work(), conf)
|
||||
|
||||
# step 1: figure out which packages are multlib (should already exist)
|
||||
# step 1: figure out which packages are multilib (should already exist)
|
||||
mlm = multilib.DevelMultilibMethod(ml_conf)
|
||||
fs_missing = set()
|
||||
with open(self.pkglist) as pkglist:
|
||||
|
|
@ -5175,6 +5181,10 @@ enabled=1
|
|||
raise koji.GenericError('multilib packages missing:\n' +
|
||||
'\n'.join(fs_missing))
|
||||
|
||||
# get rpm ids for ml pkgs
|
||||
kpkgfile = os.path.join(mldir, 'kojipkgs')
|
||||
kojipkgs = json.load(open(kpkgfile, 'r'))
|
||||
|
||||
# step 5: add dependencies to our package list
|
||||
pkgwriter = open(self.pkglist, 'a')
|
||||
for dep_path in ml_needed:
|
||||
|
|
@ -5191,11 +5201,10 @@ enabled=1
|
|||
pkgwriter.write(bnplet + '/' + bnp + '\n')
|
||||
self.logger.debug("os.symlink(%r, %r)", dep_path, dst)
|
||||
os.symlink(dep_path, dst)
|
||||
self.keypaths[bnp] = dep_path
|
||||
self.sigmap[bnp] = kojipkgs[bnp]['sigkey']
|
||||
|
||||
|
||||
def make_pkglist(self, tag_id, arch, keys, opts):
|
||||
|
||||
rpms = []
|
||||
builddirs = {}
|
||||
for a in self.compat[arch] + ('noarch',):
|
||||
|
|
@ -5228,6 +5237,7 @@ enabled=1
|
|||
preferred[rpminfo['id']] = rpminfo
|
||||
seen = set()
|
||||
fs_missing = set()
|
||||
kojipkgs = {}
|
||||
for rpminfo in preferred.values():
|
||||
if rpminfo['sigkey'] == '':
|
||||
# we're taking an unsigned rpm (--allow-unsigned)
|
||||
|
|
@ -5239,16 +5249,19 @@ enabled=1
|
|||
seen.add(os.path.basename(pkgpath))
|
||||
if not os.path.exists(pkgpath):
|
||||
fs_missing.add(pkgpath)
|
||||
# we'll raise an error below
|
||||
else:
|
||||
bnp = os.path.basename(pkgpath)
|
||||
bnplet = bnp[0].lower()
|
||||
pkglist.write(bnplet + '/' + bnp + '\n')
|
||||
koji.ensuredir(os.path.join(self.repodir, bnplet))
|
||||
self.keypaths[bnp] = pkgpath
|
||||
self.sigmap[rpminfo['id']] = rpminfo['sigkey']
|
||||
dst = os.path.join(self.repodir, bnplet, bnp)
|
||||
self.logger.debug("os.symlink(%r, %r(", pkgpath, dst)
|
||||
os.symlink(pkgpath, dst)
|
||||
kojipkgs[bnp] = rpminfo
|
||||
pkglist.close()
|
||||
self.kojipkgs = kojipkgs
|
||||
if len(fs_missing) > 0:
|
||||
raise koji.GenericError('Packages missing from the filesystem:\n' +
|
||||
'\n'.join(fs_missing))
|
||||
|
|
@ -5261,6 +5274,13 @@ enabled=1
|
|||
return pkgfile
|
||||
|
||||
|
||||
def write_kojipkgs(self):
|
||||
datafile = file(os.path.join(self.repodir, 'kojipkgs'), 'w')
|
||||
json.dump(self.kojipkgs, datafile, indent=4)
|
||||
datafile.close()
|
||||
|
||||
|
||||
|
||||
class WaitrepoTask(BaseTaskHandler):
|
||||
|
||||
Methods = ['waitrepo']
|
||||
|
|
|
|||
|
|
@ -12325,10 +12325,27 @@ class HostExports(object):
|
|||
log_error("Unable to create latest link for repo: %s" % repodir)
|
||||
koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire)
|
||||
|
||||
def signedRepoMove(self, repo_id, uploadpath, files, arch, fullpaths):
|
||||
|
||||
def signedRepoMove(self, repo_id, uploadpath, files, arch, sigmap):
|
||||
"""
|
||||
Move a signed repo into its final location
|
||||
|
||||
|
||||
Unlike normal repos (which are moved into place by repoDone), signed
|
||||
repos have all their content linked (or copied) into place.
|
||||
|
||||
repo_id - the repo to move
|
||||
uploadpath - where the uploaded files are
|
||||
files - a list of the uploaded file names
|
||||
arch - the arch of the repo
|
||||
sigmap - a dictionary rpm_id -> sig
|
||||
|
||||
The rpms from sigmap should match the contents of the uploaded pkglist
|
||||
file.
|
||||
|
||||
In sigmap, use sig=None to use the primary copy of the rpm instead of a
|
||||
signed copy.
|
||||
"""
|
||||
Very similar to repoDone, except only the uploads are completed.
|
||||
fullpaths is a dict like so: rpm file name -> sig"""
|
||||
workdir = koji.pathinfo.work()
|
||||
rinfo = repo_info(repo_id, strict=True)
|
||||
repodir = koji.pathinfo.signedrepo(repo_id, rinfo['tag_name'])
|
||||
|
|
@ -12337,6 +12354,8 @@ class HostExports(object):
|
|||
raise koji.GenericError("Repo arch directory missing: %s" % archdir)
|
||||
datadir = "%s/repodata" % archdir
|
||||
koji.ensuredir(datadir)
|
||||
|
||||
pkglist = set()
|
||||
for fn in files:
|
||||
src = "%s/%s/%s" % (workdir, uploadpath, fn)
|
||||
if fn.endswith('.drpm'):
|
||||
|
|
@ -12349,30 +12368,60 @@ class HostExports(object):
|
|||
if not os.path.exists(src):
|
||||
raise koji.GenericError("uploaded file missing: %s" % src)
|
||||
if fn.endswith('pkglist'):
|
||||
# hardlink the found rpms into the final repodir
|
||||
# TODO: properly consider split-volume functionality
|
||||
with open(src) as pkgfile:
|
||||
for pkg in pkgfile:
|
||||
pkg = os.path.basename(pkg.strip())
|
||||
rpmpath = fullpaths[pkg]
|
||||
bnp = os.path.basename(rpmpath)
|
||||
bnplet = bnp[0].lower()
|
||||
koji.ensuredir(os.path.join(archdir, bnplet))
|
||||
l_dst = os.path.join(archdir, bnplet, bnp)
|
||||
if os.path.exists(l_dst):
|
||||
logger.warning("Path exists: %s", l_dst)
|
||||
continue
|
||||
logger.debug("os.link(%r, %r)", rpmpath, l_dst)
|
||||
try:
|
||||
os.link(rpmpath, l_dst)
|
||||
except OSError, ose:
|
||||
if ose.errno == 18:
|
||||
shutil.copy2(
|
||||
rpmpath, os.path.join(archdir, bnplet, bnp))
|
||||
else:
|
||||
raise
|
||||
pkglist.add(pkg)
|
||||
safer_move(src, dst)
|
||||
|
||||
# get rpms
|
||||
build_dirs = {}
|
||||
rpmdata = {}
|
||||
for rpm_id in sigmap:
|
||||
sigkey = sigmap[rpm_id]
|
||||
rpminfo = get_rpm(rpm_id, strict=True)
|
||||
relpath = koji.pathinfo.signed(rpminfo, sigkey)
|
||||
rpminfo['_relpath'] = relpath
|
||||
if rpminfo['build_id'] in build_dirs:
|
||||
builddir = build_dirs[rpminfo['build_id']]
|
||||
else:
|
||||
binfo = get_build(rpminfo['build_id'])
|
||||
builddir = koji.pathinfo.build(binfo)
|
||||
build_dirs[rpminfo['build_id']] = builddir
|
||||
rpminfo['_fullpath'] = os.path.join(builddir, relpath)
|
||||
basename = os.path.basename(relpath)
|
||||
rpmdata[basename] = rpminfo
|
||||
|
||||
# sanity check
|
||||
for fn in rpmdata:
|
||||
if fn not in pkglist:
|
||||
raise koji.GenericError("No signature data for: %s" % fn)
|
||||
for fn in pkglist:
|
||||
if fn not in rpmdata:
|
||||
raise koji.GenericError("RPM missing from pkglist: %s" % fn)
|
||||
|
||||
for fn in rpmdata:
|
||||
# hardlink or copy the rpms into the final repodir
|
||||
# TODO: properly consider split-volume functionality
|
||||
rpminfo = rpmdata[fn]
|
||||
rpmpath = rpminfo['_fullpath']
|
||||
bnp = fn
|
||||
bnplet = bnp[0].lower()
|
||||
koji.ensuredir(os.path.join(archdir, bnplet))
|
||||
l_dst = os.path.join(archdir, bnplet, bnp)
|
||||
if os.path.exists(l_dst):
|
||||
raise koji.GenericError("File already in repo: %s", l_dst)
|
||||
logger.debug("os.link(%r, %r)", rpmpath, l_dst)
|
||||
try:
|
||||
os.link(rpmpath, l_dst)
|
||||
except OSError, ose:
|
||||
if ose.errno == 18:
|
||||
shutil.copy2(
|
||||
rpmpath, os.path.join(archdir, bnplet, bnp))
|
||||
else:
|
||||
raise
|
||||
|
||||
|
||||
def isEnabled(self):
|
||||
host = Host()
|
||||
host.verify()
|
||||
|
|
|
|||
|
|
@ -131,19 +131,21 @@ class TestSignedRepoMove(unittest.TestCase):
|
|||
with open(path, 'w') as fo:
|
||||
fo.write('%s' % fn)
|
||||
|
||||
# also a pkglist file
|
||||
# generate pkglist file and sigmap
|
||||
self.files.append('pkglist')
|
||||
plist = os.path.join(uploaddir, 'pkglist')
|
||||
# crap this is terrible -- code needs fixing
|
||||
nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34']
|
||||
self.fullpaths = {} # XXX
|
||||
self.sigmap = {}
|
||||
self.rpms = {}
|
||||
self.builds ={}
|
||||
self.key = '4c8da725'
|
||||
with open(plist, 'w') as f_pkglist:
|
||||
for nvr in nvrs:
|
||||
binfo = koji.parse_NVR(nvr)
|
||||
rpminfo = binfo.copy()
|
||||
rpminfo['arch'] = 'x86_64'
|
||||
builddir = koji.pathinfo.build(binfo)
|
||||
relpath = koji.pathinfo.rpm(rpminfo)
|
||||
relpath = koji.pathinfo.signed(rpminfo, self.key)
|
||||
path = os.path.join(builddir, relpath)
|
||||
koji.ensuredir(os.path.dirname(path))
|
||||
basename = os.path.basename(path)
|
||||
|
|
@ -152,11 +154,22 @@ class TestSignedRepoMove(unittest.TestCase):
|
|||
f_pkglist.write(path)
|
||||
f_pkglist.write('\n')
|
||||
self.expected.append('x86_64/%s/%s' % (basename[0], basename))
|
||||
self.fullpaths[basename] = path # XXX
|
||||
build_id = len(self.builds) + 10000
|
||||
rpm_id = len(self.rpms) + 20000
|
||||
binfo['id'] = build_id
|
||||
rpminfo['build_id'] = build_id
|
||||
rpminfo['id'] = rpm_id
|
||||
self.builds[build_id] = binfo
|
||||
self.rpms[rpm_id] = rpminfo
|
||||
self.sigmap[rpm_id] = self.key
|
||||
|
||||
# mocks
|
||||
self.repo_info = mock.patch('kojihub.repo_info').start()
|
||||
self.repo_info.return_value = self.rinfo.copy()
|
||||
self.get_rpm = mock.patch('kojihub.get_rpm').start()
|
||||
self.get_build = mock.patch('kojihub.get_build').start()
|
||||
self.get_rpm.side_effect = self.our_get_rpm
|
||||
self.get_build.side_effect = self.our_get_build
|
||||
|
||||
|
||||
def tearDown(self):
|
||||
|
|
@ -164,10 +177,18 @@ class TestSignedRepoMove(unittest.TestCase):
|
|||
shutil.rmtree(self.topdir)
|
||||
|
||||
|
||||
def our_get_rpm(self, rpminfo, strict=False, multi=False):
|
||||
return self.rpms[rpminfo]
|
||||
|
||||
|
||||
def our_get_build(self, buildInfo, strict=False):
|
||||
return self.builds[buildInfo]
|
||||
|
||||
|
||||
def test_signedRepoMove(self):
|
||||
exports = kojihub.HostExports()
|
||||
exports.signedRepoMove(self.rinfo['id'], self.uploadpath,
|
||||
list(self.files), self.arch, self.fullpaths)
|
||||
list(self.files), self.arch, self.sigmap)
|
||||
# check result
|
||||
repodir = self.topdir + '/repos-signed/%(tag_name)s/%(id)s' % self.rinfo
|
||||
for relpath in self.expected:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue