PR#914: dist repo updates
Merges #914 https://pagure.io/koji/pull-request/914 Fixes: #409 https://pagure.io/koji/issue/409 dist-repos are missing critical functionality Fixes: #457 https://pagure.io/koji/issue/457 creating dist-repos can expire regular (non-dist) repos for same tag
This commit is contained in:
Mike McLean
commit
18600b24c1
7 changed files with 393 additions and 191 deletions
301
builder/kojid
301
builder/kojid
|
|
@ -5055,7 +5055,7 @@ class CreaterepoTask(BaseTaskHandler):
|
|||
self.session.uploadWrapper('%s/%s' % (self.datadir, f), uploadpath, f)
|
||||
return [uploadpath, files]
|
||||
|
||||
def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo, oldpkgs=None):
|
||||
def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):
|
||||
koji.ensuredir(self.outdir)
|
||||
if self.options.use_createrepo_c:
|
||||
cmd = ['/usr/bin/createrepo_c']
|
||||
|
|
@ -5082,11 +5082,6 @@ class CreaterepoTask(BaseTaskHandler):
|
|||
cmd.append('--update')
|
||||
if self.options.createrepo_skip_stat:
|
||||
cmd.append('--skip-stat')
|
||||
if oldpkgs:
|
||||
# generate delta-rpms
|
||||
cmd.append('--deltas')
|
||||
for op_dir in oldpkgs:
|
||||
cmd.extend(['--oldpackagedirs', op_dir])
|
||||
# note: we can't easily use a cachedir because we do not have write
|
||||
# permission. The good news is that with --update we won't need to
|
||||
# be scanning many rpms.
|
||||
|
|
@ -5139,10 +5134,9 @@ class NewDistRepoTask(BaseTaskHandler):
|
|||
|
||||
def handler(self, tag, repo_id, keys, task_opts):
|
||||
tinfo = self.session.getTag(tag, strict=True, event=task_opts['event'])
|
||||
path = koji.pathinfo.distrepo(repo_id, tinfo['name'])
|
||||
if len(task_opts['arch']) == 0:
|
||||
arches = tinfo['arches'] or ''
|
||||
task_opts['arch'] = arches.split()
|
||||
arches = tinfo['arches'] or ''
|
||||
task_opts['arch'] = arches.split()
|
||||
if len(task_opts['arch']) == 0:
|
||||
raise koji.GenericError('No arches specified nor for the tag!')
|
||||
subtasks = {}
|
||||
|
|
@ -5162,13 +5156,12 @@ class NewDistRepoTask(BaseTaskHandler):
|
|||
method='createdistrepo', arglist=arglist, label=arch,
|
||||
parent=self.id, arch='noarch')
|
||||
if len(subtasks) > 0 and task_opts['multilib']:
|
||||
results = self.wait(subtasks.values(), all=True, failany=True)
|
||||
self.wait(subtasks.values(), all=True, failany=True)
|
||||
for arch in arch32s:
|
||||
# move the 32-bit task output to the final resting place
|
||||
# so the 64-bit arches can use it for multilib
|
||||
upload, files, sigmap = results[subtasks[arch]]
|
||||
self.session.host.distRepoMove(
|
||||
repo_id, upload, files, arch, sigmap)
|
||||
upload_dir = koji.pathinfo.taskrelpath(subtasks[arch])
|
||||
self.session.host.distRepoMove(repo_id, upload_dir, arch)
|
||||
for arch in canonArches:
|
||||
# do the other arches
|
||||
if arch not in arch32s:
|
||||
|
|
@ -5177,23 +5170,18 @@ class NewDistRepoTask(BaseTaskHandler):
|
|||
method='createdistrepo', arglist=arglist, label=arch,
|
||||
parent=self.id, arch='noarch')
|
||||
# wait for 64-bit subtasks to finish
|
||||
data = {}
|
||||
results = self.wait(subtasks.values(), all=True, failany=True)
|
||||
self.wait(subtasks.values(), all=True, failany=True)
|
||||
for (arch, task_id) in subtasks.iteritems():
|
||||
data[arch] = results[task_id]
|
||||
self.logger.debug("DEBUG: %r : %r " % (arch, data[arch]))
|
||||
if task_opts['multilib'] and arch in arch32s:
|
||||
# already moved above
|
||||
continue
|
||||
#else
|
||||
upload, files, sigmap = results[subtasks[arch]]
|
||||
self.session.host.distRepoMove(
|
||||
repo_id, upload, files, arch, sigmap)
|
||||
self.session.host.repoDone(repo_id, data, expire=False)
|
||||
upload_dir = koji.pathinfo.taskrelpath(subtasks[arch])
|
||||
self.session.host.distRepoMove(repo_id, upload_dir, arch)
|
||||
self.session.host.repoDone(repo_id, {}, expire=False)
|
||||
return 'Dist repository #%s successfully generated' % repo_id
|
||||
|
||||
|
||||
class createDistRepoTask(CreaterepoTask):
|
||||
class createDistRepoTask(BaseTaskHandler):
|
||||
Methods = ['createdistrepo']
|
||||
_taskWeight = 1.5
|
||||
|
||||
|
|
@ -5223,17 +5211,17 @@ class createDistRepoTask(CreaterepoTask):
|
|||
self.rinfo = self.session.repoInfo(repo_id, strict=True)
|
||||
if self.rinfo['state'] != koji.REPO_INIT:
|
||||
raise koji.GenericError("Repo %(id)s not in INIT state (got %(state)s)" % self.rinfo)
|
||||
self.repo_id = self.rinfo['id']
|
||||
self.pathinfo = koji.PathInfo(self.options.topdir)
|
||||
groupdata = os.path.join(
|
||||
self.pathinfo.distrepo(repo_id, self.rinfo['tag_name']),
|
||||
koji.pathinfo.distrepo(repo_id, self.rinfo['tag_name']),
|
||||
'groups', 'comps.xml')
|
||||
#set up our output dir
|
||||
|
||||
# set up our output dir
|
||||
self.repodir = '%s/repo' % self.workdir
|
||||
self.repo_files = []
|
||||
koji.ensuredir(self.repodir)
|
||||
self.outdir = self.repodir # workaround create_local_repo use
|
||||
self.datadir = '%s/repodata' % self.repodir
|
||||
self.sigmap = {}
|
||||
self.subrepos = set()
|
||||
|
||||
# gather oldpkgs data if delta option in use
|
||||
oldpkgs = []
|
||||
if opts.get('delta'):
|
||||
# should be a list of repo ids to delta against
|
||||
|
|
@ -5246,63 +5234,136 @@ class createDistRepoTask(CreaterepoTask):
|
|||
path = koji.pathinfo.distrepo(repo_id, oldrepo['tag_name'])
|
||||
if not os.path.exists(path):
|
||||
raise koji.GenericError('Base drpm repo missing: %s' % path)
|
||||
# note: since we're using the top level dir, this will handle
|
||||
# split repos as well
|
||||
oldpkgs.append(path)
|
||||
|
||||
# sort out our package list(s)
|
||||
self.uploadpath = self.getUploadDir()
|
||||
self.pkglist = self.make_pkglist(tag, arch, keys, opts)
|
||||
self.get_rpms(tag, arch, keys, opts)
|
||||
if opts['multilib'] and rpmUtils.arch.isMultiLibArch(arch):
|
||||
self.do_multilib(arch, self.archmap[arch], opts['multilib'])
|
||||
self.split_pkgs(opts)
|
||||
self.write_kojipkgs()
|
||||
self.logger.debug('package list is %s' % self.pkglist)
|
||||
self.session.uploadWrapper(self.pkglist, self.uploadpath,
|
||||
os.path.basename(self.pkglist))
|
||||
if os.path.getsize(self.pkglist) == 0:
|
||||
self.pkglist = None
|
||||
self.create_local_repo(self.rinfo, arch, self.pkglist, groupdata, None, oldpkgs=oldpkgs)
|
||||
if self.pkglist is None:
|
||||
fo = file(os.path.join(self.datadir, "EMPTY_REPO"), 'w')
|
||||
fo.write("This repo is empty because its tag has no content for this arch\n")
|
||||
fo.close()
|
||||
files = ['pkglist', 'kojipkgs']
|
||||
for f in os.listdir(self.datadir):
|
||||
files.append(f)
|
||||
self.session.uploadWrapper('%s/%s' % (self.datadir, f),
|
||||
self.uploadpath, f)
|
||||
if opts['delta']:
|
||||
ddir = os.path.join(self.repodir, 'drpms')
|
||||
for f in os.listdir(ddir):
|
||||
files.append(f)
|
||||
self.session.uploadWrapper('%s/%s' % (ddir, f),
|
||||
self.uploadpath, f)
|
||||
return [self.uploadpath, files, self.sigmap.items()]
|
||||
self.write_pkglist()
|
||||
self.link_pkgs()
|
||||
|
||||
# generate the repodata
|
||||
self.do_createrepo(self.repodir, '%s/pkglist' % self.repodir,
|
||||
groupdata, oldpkgs=oldpkgs)
|
||||
for subrepo in self.subrepos:
|
||||
self.do_createrepo(
|
||||
'%s/%s' % (self.repodir, subrepo),
|
||||
'%s/%s/pkglist' % (self.repodir, subrepo),
|
||||
groupdata, oldpkgs=oldpkgs,
|
||||
logname='createrepo_%s' % subrepo)
|
||||
if len(self.kojipkgs) == 0:
|
||||
fn = os.path.join(self.repodir, "repodata", "EMPTY_REPO")
|
||||
with open(fn, 'w') as fp:
|
||||
fp.write("This repo is empty because its tag has no content "
|
||||
"for this arch\n")
|
||||
|
||||
# upload repo files
|
||||
self.upload_repo()
|
||||
self.upload_repo_manifest()
|
||||
|
||||
def upload_repo_file(self, relpath):
|
||||
"""Upload a file from the repo
|
||||
|
||||
relpath should be relative to self.repodir
|
||||
"""
|
||||
|
||||
localpath = '%s/%s' % (self.repodir, relpath)
|
||||
reldir = os.path.dirname(relpath)
|
||||
if reldir:
|
||||
uploadpath = "%s/%s" % (self.uploadpath, reldir)
|
||||
fn = os.path.basename(relpath)
|
||||
else:
|
||||
uploadpath = self.uploadpath
|
||||
fn = relpath
|
||||
self.session.uploadWrapper(localpath, uploadpath, fn)
|
||||
self.repo_files.append(relpath)
|
||||
|
||||
def upload_repo(self):
|
||||
"""Traverse the repo and upload needed files
|
||||
|
||||
We omit the symlinks we made for the rpms
|
||||
"""
|
||||
for dirpath, dirs, files in os.walk(self.repodir):
|
||||
reldir = os.path.relpath(dirpath, self.repodir)
|
||||
for filename in files:
|
||||
path = "%s/%s" % (dirpath, filename)
|
||||
if os.path.islink(path):
|
||||
continue
|
||||
relpath = "%s/%s" % (reldir, filename)
|
||||
self.upload_repo_file(relpath)
|
||||
|
||||
def upload_repo_manifest(self):
|
||||
"""Upload a list of the repo files we've uploaded"""
|
||||
fn = '%s/repo_manifest' % self.workdir
|
||||
with open(fn, 'w') as fp:
|
||||
json.dump(self.repo_files, fp, indent=4)
|
||||
self.session.uploadWrapper(fn, self.uploadpath)
|
||||
|
||||
def do_createrepo(self, repodir, pkglist, groupdata, oldpkgs=None, logname=None):
|
||||
"""Run createrepo
|
||||
|
||||
This is derived from CreaterepoTask.create_local_repo, but adapted to
|
||||
our requirements here
|
||||
"""
|
||||
koji.ensuredir(repodir)
|
||||
if self.options.use_createrepo_c:
|
||||
cmd = ['/usr/bin/createrepo_c']
|
||||
else:
|
||||
cmd = ['/usr/bin/createrepo']
|
||||
cmd.extend(['-vd', '-i', pkglist])
|
||||
if groupdata and os.path.isfile(groupdata):
|
||||
cmd.extend(['-g', groupdata])
|
||||
# TODO: can we recycle data (with --update) as in create_local_repo?
|
||||
if oldpkgs:
|
||||
# generate delta-rpms
|
||||
cmd.append('--deltas')
|
||||
for op_dir in oldpkgs:
|
||||
cmd.extend(['--oldpackagedirs', op_dir])
|
||||
cmd.append(repodir)
|
||||
|
||||
if logname is None:
|
||||
logname = 'createrepo'
|
||||
logfile = '%s/%s.log' % (self.workdir, logname)
|
||||
status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
|
||||
if not isSuccess(status):
|
||||
raise koji.GenericError('failed to create repo: %s' \
|
||||
% parseStatus(status, ' '.join(cmd)))
|
||||
|
||||
def do_multilib(self, arch, ml_arch, conf):
|
||||
self.repo_id = self.rinfo['id']
|
||||
pathinfo = koji.PathInfo(self.options.topdir)
|
||||
repodir = pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])
|
||||
repodir = koji.pathinfo.distrepo(self.rinfo['id'], self.rinfo['tag_name'])
|
||||
mldir = os.path.join(repodir, koji.canonArch(ml_arch))
|
||||
ml_true = set() # multilib packages we need to include before depsolve
|
||||
ml_conf = os.path.join(self.pathinfo.work(), conf)
|
||||
ml_true = set() # multilib packages we need to include before depsolve
|
||||
ml_conf = os.path.join(koji.pathinfo.work(), conf)
|
||||
|
||||
# read pkgs data from multilib repo
|
||||
ml_pkgfile = os.path.join(mldir, 'kojipkgs')
|
||||
ml_pkgs = json.load(open(ml_pkgfile, 'r'))
|
||||
|
||||
# step 1: figure out which packages are multilib (should already exist)
|
||||
mlm = multilib.DevelMultilibMethod(ml_conf)
|
||||
fs_missing = set()
|
||||
with open(self.pkglist) as pkglist:
|
||||
for pkg in pkglist:
|
||||
ppath = os.path.join(self.repodir, pkg.strip())
|
||||
for bnp in self.kojipkgs:
|
||||
rpminfo = self.kojipkgs[bnp]
|
||||
ppath = rpminfo['_pkgpath']
|
||||
po = yum.packages.YumLocalPackage(filename=ppath)
|
||||
if mlm.select(po) and arch in self.archmap:
|
||||
if mlm.select(po):
|
||||
# we need a multilib package to be included
|
||||
# we assume the same signature level is available
|
||||
# XXX: what is a subarchitecture is the right answer?
|
||||
pl_path = pkg.replace(arch, self.archmap[arch]).strip()
|
||||
# assume this exists in the task results for the ml arch
|
||||
real_path = os.path.join(mldir, pl_path)
|
||||
if not os.path.exists(real_path):
|
||||
self.logger.error('%s (multilib) is not on the filesystem' % real_path)
|
||||
fs_missing.add(real_path)
|
||||
ml_bnp = bnp.replace(arch, self.archmap[arch])
|
||||
ml_path = os.path.join(mldir, ml_bnp[0].lower(), ml_bnp)
|
||||
# ^ XXX - should actually generate this
|
||||
if ml_bnp not in ml_pkgs:
|
||||
# not in our multilib repo
|
||||
self.logger.error('%s (multilib) is not on the filesystem' % ml_path)
|
||||
fs_missing.add(ml_path)
|
||||
# we defer failure so can report all the missing deps
|
||||
continue
|
||||
ml_true.add(real_path)
|
||||
ml_true.add(ml_path)
|
||||
|
||||
# step 2: set up architectures for yum configuration
|
||||
self.logger.info("Resolving multilib for %s using method devel" % arch)
|
||||
|
|
@ -5392,29 +5453,22 @@ enabled=1
|
|||
raise koji.GenericError('multilib packages missing. '
|
||||
'See missing_multilib.log')
|
||||
|
||||
# get rpm ids for ml pkgs
|
||||
kpkgfile = os.path.join(mldir, 'kojipkgs')
|
||||
kojipkgs = json.load(open(kpkgfile, 'r'))
|
||||
|
||||
# step 5: add dependencies to our package list
|
||||
pkgwriter = open(self.pkglist, 'a')
|
||||
# step 5: update kojipkgs
|
||||
for dep_path in ml_needed:
|
||||
tspkg = ml_needed[dep_path]
|
||||
bnp = os.path.basename(dep_path)
|
||||
bnplet = bnp[0].lower()
|
||||
koji.ensuredir(os.path.join(self.repodir, bnplet))
|
||||
dst = os.path.join(self.repodir, bnplet, bnp)
|
||||
if os.path.exists(dst):
|
||||
if bnp in self.kojipkgs:
|
||||
# we expect duplication with noarch, but not other arches
|
||||
if tspkg.arch != 'noarch':
|
||||
self.logger.warning("Path exists: %r", dst)
|
||||
self.logger.warning("Multilib duplicate: %s", bnp)
|
||||
continue
|
||||
pkgwriter.write(bnplet + '/' + bnp + '\n')
|
||||
self.logger.debug("os.symlink(%r, %r)", dep_path, dst)
|
||||
os.symlink(dep_path, dst)
|
||||
rpminfo = kojipkgs[bnp]
|
||||
self.sigmap[rpminfo['id']] = rpminfo['sigkey']
|
||||
|
||||
rpminfo = ml_pkgs[bnp].copy()
|
||||
# fix _pkgpath, which comes from another task and could be wrong
|
||||
# for us
|
||||
# TODO: would be better if we could use the proper path here
|
||||
rpminfo['_pkgpath'] = dep_path
|
||||
rpminfo['_multilib'] = True
|
||||
self.kojipkgs[bnp] = rpminfo
|
||||
|
||||
def pick_key(self, keys, avail_keys):
|
||||
best = None
|
||||
|
|
@ -5430,21 +5484,20 @@ enabled=1
|
|||
best_idx = idx
|
||||
return best
|
||||
|
||||
|
||||
def make_pkglist(self, tag_id, arch, keys, opts):
|
||||
def get_rpms(self, tag_id, arch, keys, opts):
|
||||
# get the rpm data
|
||||
rpms = []
|
||||
builddirs = {}
|
||||
for a in self.compat[arch] + ('noarch',):
|
||||
for a in self.compat[arch]:
|
||||
# note: self.compat includes noarch for non-src already
|
||||
rpm_iter, builds = self.session.listTaggedRPMS(tag_id,
|
||||
event=opts['event'], arch=a, latest=opts['latest'],
|
||||
inherit=opts['inherit'], rpmsigs=True)
|
||||
for build in builds:
|
||||
builddirs[build['id']] = self.pathinfo.build(build)
|
||||
builddirs[build['id']] = koji.pathinfo.build(build)
|
||||
rpms += list(rpm_iter)
|
||||
|
||||
# index by id and key
|
||||
preferred = {}
|
||||
rpm_idx = {}
|
||||
for rpminfo in rpms:
|
||||
sigidx = rpm_idx.setdefault(rpminfo['id'], {})
|
||||
|
|
@ -5464,9 +5517,7 @@ enabled=1
|
|||
else:
|
||||
selected[rpm_id] = rpm_idx[rpm_id][best_key]
|
||||
|
||||
#generate pkglist files
|
||||
pkgfile = os.path.join(self.repodir, 'pkglist')
|
||||
pkglist = file(pkgfile, 'w')
|
||||
# generate kojipkgs data and note missing files
|
||||
fs_missing = []
|
||||
sig_missing = []
|
||||
kojipkgs = {}
|
||||
|
|
@ -5478,25 +5529,18 @@ enabled=1
|
|||
continue
|
||||
# use the primary copy, if allowed (checked below)
|
||||
pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']],
|
||||
self.pathinfo.rpm(rpminfo))
|
||||
koji.pathinfo.rpm(rpminfo))
|
||||
else:
|
||||
# use the signed copy
|
||||
pkgpath = '%s/%s' % (builddirs[rpminfo['build_id']],
|
||||
self.pathinfo.signed(rpminfo, rpminfo['sigkey']))
|
||||
koji.pathinfo.signed(rpminfo, rpminfo['sigkey']))
|
||||
if not os.path.exists(pkgpath):
|
||||
fs_missing.append(pkgpath)
|
||||
# we'll raise an error below
|
||||
else:
|
||||
bnp = os.path.basename(pkgpath)
|
||||
bnplet = bnp[0].lower()
|
||||
pkglist.write(bnplet + '/' + bnp + '\n')
|
||||
koji.ensuredir(os.path.join(self.repodir, bnplet))
|
||||
self.sigmap[rpminfo['id']] = rpminfo['sigkey']
|
||||
dst = os.path.join(self.repodir, bnplet, bnp)
|
||||
self.logger.debug("os.symlink(%r, %r(", pkgpath, dst)
|
||||
os.symlink(pkgpath, dst)
|
||||
rpminfo['_pkgpath'] = pkgpath
|
||||
kojipkgs[bnp] = rpminfo
|
||||
pkglist.close()
|
||||
self.kojipkgs = kojipkgs
|
||||
|
||||
# report problems
|
||||
|
|
@ -5534,19 +5578,54 @@ enabled=1
|
|||
and not opts['allow_missing_signatures']):
|
||||
raise koji.GenericError('Unsigned packages found. See '
|
||||
'missing_signatures.log')
|
||||
return pkgfile
|
||||
|
||||
def link_pkgs(self):
|
||||
for bnp in self.kojipkgs:
|
||||
bnplet = bnp[0].lower()
|
||||
ddir = os.path.join(self.repodir, 'Packages', bnplet)
|
||||
koji.ensuredir(ddir)
|
||||
dst = os.path.join(ddir, bnp)
|
||||
pkgpath = self.kojipkgs[bnp]['_pkgpath']
|
||||
self.logger.debug("os.symlink(%r, %r(", pkgpath, dst)
|
||||
os.symlink(pkgpath, dst)
|
||||
|
||||
def split_pkgs(self, opts):
|
||||
'''Direct rpms to subrepos if needed'''
|
||||
for rpminfo in self.kojipkgs.values():
|
||||
if opts['split_debuginfo'] and koji.is_debuginfo(rpminfo['name']):
|
||||
rpminfo['_subrepo'] = 'debug'
|
||||
self.subrepos.add('debug')
|
||||
|
||||
def write_pkglist(self):
|
||||
pkgs = []
|
||||
subrepo_pkgs = {}
|
||||
for bnp in self.kojipkgs:
|
||||
rpminfo = self.kojipkgs[bnp]
|
||||
bnplet = bnp[0].lower()
|
||||
subrepo = rpminfo.get('_subrepo')
|
||||
if subrepo:
|
||||
# note the ../
|
||||
subrepo_pkgs.setdefault(subrepo, []).append(
|
||||
'../Packages/%s/%s\n' % (bnplet, bnp))
|
||||
else:
|
||||
pkgs.append('Packages/%s/%s\n' % (bnplet, bnp))
|
||||
|
||||
with open('%s/pkglist' % self.repodir, 'w') as fo:
|
||||
for line in pkgs:
|
||||
fo.write(line)
|
||||
for subrepo in subrepo_pkgs:
|
||||
koji.ensuredir('%s/%s' % (self.repodir, subrepo))
|
||||
with open('%s/%s/pkglist' % (self.repodir, subrepo), 'w') as fo:
|
||||
for line in subrepo_pkgs[subrepo]:
|
||||
fo.write(line)
|
||||
|
||||
def write_kojipkgs(self):
|
||||
filename = os.path.join(self.repodir, 'kojipkgs')
|
||||
datafile = file(filename, 'w')
|
||||
try:
|
||||
json.dump(self.kojipkgs, datafile, indent=4)
|
||||
json.dump(self.kojipkgs, datafile, indent=4, sort_keys=True)
|
||||
finally:
|
||||
datafile.close()
|
||||
# and upload too
|
||||
self.session.uploadWrapper(filename, self.uploadpath, 'kojipkgs')
|
||||
|
||||
|
||||
|
||||
class WaitrepoTask(BaseTaskHandler):
|
||||
|
|
|
|||
|
|
@ -6911,10 +6911,13 @@ def handle_dist_repo(options, session, args):
|
|||
default=False,
|
||||
help=_('For RPMs not signed with a desired key, fall back to the '
|
||||
'primary copy'))
|
||||
parser.add_option("--arch", action='append', default=[],
|
||||
parser.add_option("-a", "--arch", action='append', default=[],
|
||||
help=_("Indicate an architecture to consider. The default is all " +
|
||||
"architectures associated with the given tag. This option may " +
|
||||
"be specified multiple times."))
|
||||
parser.add_option("--with-src", action='store_true', help='Also generate a src repo')
|
||||
parser.add_option("--split-debuginfo", action='store_true', default=False,
|
||||
help='Split debuginfo info a separate repo for each arch')
|
||||
parser.add_option('--comps', help='Include a comps file in the repodata')
|
||||
parser.add_option('--delta-rpms', metavar='REPO',default=[],
|
||||
action='append',
|
||||
|
|
@ -6922,7 +6925,7 @@ def handle_dist_repo(options, session, args):
|
|||
'or the name of a tag that has a dist repo. May be specified '
|
||||
'multiple times.'))
|
||||
parser.add_option('--event', type='int',
|
||||
help=_('create a dist repository based on a Brew event'))
|
||||
help=_('Use tag content at event'))
|
||||
parser.add_option('--non-latest', dest='latest', default=True,
|
||||
action='store_false', help='Include older builds, not just the latest')
|
||||
parser.add_option('--multilib', default=None, metavar="CONFIG",
|
||||
|
|
@ -6996,9 +6999,10 @@ def handle_dist_repo(options, session, args):
|
|||
task_opts.multilib = os.path.join(stuffdir,
|
||||
os.path.basename(task_opts.multilib))
|
||||
print('')
|
||||
for f in ('noarch', 'src'):
|
||||
if f in task_opts.arch:
|
||||
task_opts.arch.remove(f)
|
||||
if 'noarch' in task_opts.arch:
|
||||
task_opts.arch.remove('noarch')
|
||||
if task_opts.with_src and 'src' not in task_opts.arch:
|
||||
task_opts.arch.append('src')
|
||||
opts = {
|
||||
'arch': task_opts.arch,
|
||||
'comps': task_opts.comps,
|
||||
|
|
@ -7007,6 +7011,7 @@ def handle_dist_repo(options, session, args):
|
|||
'inherit': not task_opts.noinherit,
|
||||
'latest': task_opts.latest,
|
||||
'multilib': task_opts.multilib,
|
||||
'split_debuginfo': task_opts.split_debuginfo,
|
||||
'skip_missing_signatures': task_opts.skip_missing_signatures,
|
||||
'allow_missing_signatures': task_opts.allow_missing_signatures
|
||||
}
|
||||
|
|
|
|||
151
hub/kojihub.py
151
hub/kojihub.py
|
|
@ -2470,7 +2470,7 @@ def dist_repo_init(tag, keys, task_opts):
|
|||
tinfo = get_tag(tag, strict=True)
|
||||
tag_id = tinfo['id']
|
||||
event = task_opts.get('event')
|
||||
arches = set([koji.canonArch(a) for a in task_opts['arch']])
|
||||
arches = list(set([koji.canonArch(a) for a in task_opts['arch']]))
|
||||
# note: we need to match args from the other preRepoInit callback
|
||||
koji.plugin.run_callbacks('preRepoInit', tag=tinfo, with_src=False,
|
||||
with_debuginfo=False, event=event, repo_id=None,
|
||||
|
|
@ -2493,7 +2493,9 @@ def dist_repo_init(tag, keys, task_opts):
|
|||
task_opts['comps']), groupsdir + '/comps.xml')
|
||||
# note: we need to match args from the other postRepoInit callback
|
||||
koji.plugin.run_callbacks('postRepoInit', tag=tinfo, with_src=False,
|
||||
with_debuginfo=False, event=event, repo_id=repo_id)
|
||||
with_debuginfo=False, event=event, repo_id=repo_id,
|
||||
dist=True, keys=keys, arches=arches, task_opts=task_opts,
|
||||
repodir=repodir)
|
||||
return repo_id, event
|
||||
|
||||
|
||||
|
|
@ -2550,15 +2552,23 @@ def repo_delete(repo_id):
|
|||
repo_set_state(repo_id, koji.REPO_DELETED)
|
||||
return len(references)
|
||||
|
||||
def repo_expire_older(tag_id, event_id):
|
||||
"""Expire repos for tag older than event"""
|
||||
|
||||
def repo_expire_older(tag_id, event_id, dist=None):
|
||||
"""Expire repos for tag older than event
|
||||
|
||||
If dist is not None, then only expire repos with the given dist value
|
||||
"""
|
||||
st_ready = koji.REPO_READY
|
||||
st_expired = koji.REPO_EXPIRED
|
||||
q = """UPDATE repo SET state=%(st_expired)i
|
||||
WHERE tag_id = %(tag_id)i
|
||||
AND create_event < %(event_id)i
|
||||
AND state = %(st_ready)i"""
|
||||
_dml(q, locals())
|
||||
clauses=['tag_id = %(tag_id)s',
|
||||
'create_event < %(event_id)s',
|
||||
'state = %(st_ready)s']
|
||||
if dist is not None:
|
||||
dist = bool(dist)
|
||||
clauses.append('dist = %(dist)s')
|
||||
update = UpdateProcessor('repo', values=locals(), clauses=clauses)
|
||||
update.set(state=koji.REPO_EXPIRED)
|
||||
update.execute()
|
||||
|
||||
|
||||
def repo_references(repo_id):
|
||||
"""Return a list of buildroots that reference the repo"""
|
||||
|
|
@ -12564,14 +12574,21 @@ class HostExports(object):
|
|||
safer_move(filepath, dst)
|
||||
|
||||
def repoDone(self, repo_id, data, expire=False):
|
||||
"""Move repo data into place, mark as ready, and expire earlier repos
|
||||
"""Finalize a repo
|
||||
|
||||
repo_id: the id of the repo
|
||||
data: a dictionary of the form { arch: (uploadpath, files), ...}
|
||||
expire(optional): if set to true, mark the repo expired immediately*
|
||||
data: a dictionary of repo files in the form:
|
||||
{ arch: [uploadpath, [file1, file2, ...]], ...}
|
||||
expire: if set to true, mark the repo expired immediately [*]
|
||||
|
||||
If this is a dist repo, also hardlink the rpms in the final
|
||||
directory.
|
||||
Actions:
|
||||
* Move uploaded repo files into place
|
||||
* Mark repo ready
|
||||
* Expire earlier repos
|
||||
* Move/create 'latest' symlink
|
||||
|
||||
For dist repos, the move step is skipped (that is handled in
|
||||
distRepoMove).
|
||||
|
||||
* This is used when a repo from an older event is generated
|
||||
"""
|
||||
|
|
@ -12605,7 +12622,7 @@ class HostExports(object):
|
|||
return
|
||||
#else:
|
||||
repo_ready(repo_id)
|
||||
repo_expire_older(rinfo['tag_id'], rinfo['create_event'])
|
||||
repo_expire_older(rinfo['tag_id'], rinfo['create_event'], rinfo['dist'])
|
||||
|
||||
#make a latest link
|
||||
if rinfo['dist']:
|
||||
|
|
@ -12623,25 +12640,22 @@ class HostExports(object):
|
|||
koji.plugin.run_callbacks('postRepoDone', repo=rinfo, data=data, expire=expire)
|
||||
|
||||
|
||||
def distRepoMove(self, repo_id, uploadpath, files, arch, sigmap):
|
||||
def distRepoMove(self, repo_id, uploadpath, arch):
|
||||
"""
|
||||
Move a dist repo into its final location
|
||||
Move one arch of a dist repo into its final location
|
||||
|
||||
|
||||
Unlike normal repos (which are moved into place by repoDone), dist
|
||||
repos have all their content linked (or copied) into place.
|
||||
Unlike normal repos, dist repos have all their content linked (or
|
||||
copied) into place.
|
||||
|
||||
repo_id - the repo to move
|
||||
uploadpath - where the uploaded files are
|
||||
files - a list of the uploaded file names
|
||||
arch - the arch of the repo
|
||||
sigmap - a list of [rpm_id, sig] pairs
|
||||
|
||||
The rpms from sigmap should match the contents of the uploaded pkglist
|
||||
file.
|
||||
uploadpath should contain a repo_manifest file
|
||||
|
||||
In sigmap, use sig=None to use the primary copy of the rpm instead of a
|
||||
signed copy.
|
||||
The uploaded files should include:
|
||||
- kojipkgs: json file with information about the component rpms
|
||||
- repo metadata files
|
||||
"""
|
||||
host = Host()
|
||||
host.verify()
|
||||
|
|
@ -12651,33 +12665,48 @@ class HostExports(object):
|
|||
archdir = "%s/%s" % (repodir, koji.canonArch(arch))
|
||||
if not os.path.isdir(archdir):
|
||||
raise koji.GenericError("Repo arch directory missing: %s" % archdir)
|
||||
datadir = "%s/repodata" % archdir
|
||||
koji.ensuredir(datadir)
|
||||
repo_state = koji.REPO_STATES[rinfo['state']]
|
||||
if repo_state != 'INIT':
|
||||
raise koji.GenericError('Repo is in state: %s' % repo_state)
|
||||
|
||||
pkglist = set()
|
||||
for fn in files:
|
||||
src = "%s/%s/%s" % (workdir, uploadpath, fn)
|
||||
if fn.endswith('.drpm'):
|
||||
koji.ensuredir(os.path.join(archdir, 'drpms'))
|
||||
dst = "%s/drpms/%s" % (archdir, fn)
|
||||
elif fn.endswith('pkglist') or fn.endswith('kojipkgs'):
|
||||
dst = '%s/%s' % (archdir, fn)
|
||||
else:
|
||||
dst = "%s/%s" % (datadir, fn)
|
||||
# read manifest
|
||||
fn = '%s/%s/repo_manifest' % (workdir, uploadpath)
|
||||
if not os.path.isfile(fn):
|
||||
raise koji.GenericError('Missing repo manifest')
|
||||
with open(fn) as fp:
|
||||
files = json.load(fp)
|
||||
|
||||
# Read package data
|
||||
fn = '%s/%s/kojipkgs' % (workdir, uploadpath)
|
||||
if not os.path.isfile(fn):
|
||||
raise koji.GenericError('Missing kojipkgs file')
|
||||
with open(fn) as fp:
|
||||
kojipkgs = json.load(fp)
|
||||
|
||||
# Figure out where to send the uploaded files
|
||||
file_moves = []
|
||||
for relpath in files:
|
||||
src = "%s/%s/%s" % (workdir, uploadpath, relpath)
|
||||
dst = "%s/%s" % (archdir, relpath)
|
||||
if not os.path.exists(src):
|
||||
raise koji.GenericError("uploaded file missing: %s" % src)
|
||||
if fn.endswith('pkglist'):
|
||||
with open(src) as pkgfile:
|
||||
for pkg in pkgfile:
|
||||
pkg = os.path.basename(pkg.strip())
|
||||
pkglist.add(pkg)
|
||||
safer_move(src, dst)
|
||||
file_moves.append([src, dst])
|
||||
|
||||
# get rpms
|
||||
build_dirs = {}
|
||||
rpmdata = {}
|
||||
for rpm_id, sigkey in sigmap:
|
||||
rpminfo = get_rpm(rpm_id, strict=True)
|
||||
rpm_check_keys = ['name', 'version', 'release', 'arch', 'epoch',
|
||||
'size', 'payloadhash', 'build_id']
|
||||
for bnp in kojipkgs:
|
||||
rpminfo = kojipkgs[bnp]
|
||||
rpm_id = rpminfo['id']
|
||||
sigkey = rpminfo['sigkey']
|
||||
_rpminfo = get_rpm(rpm_id, strict=True)
|
||||
for key in rpm_check_keys:
|
||||
if key not in rpminfo or rpminfo[key] != _rpminfo[key]:
|
||||
raise koji.GenericError(
|
||||
'kojipkgs entry does not match db: file %s, key %s'
|
||||
% (bnp, key))
|
||||
if sigkey is None or sigkey == '':
|
||||
relpath = koji.pathinfo.rpm(rpminfo)
|
||||
else:
|
||||
|
|
@ -12690,26 +12719,25 @@ class HostExports(object):
|
|||
builddir = koji.pathinfo.build(binfo)
|
||||
build_dirs[rpminfo['build_id']] = builddir
|
||||
rpminfo['_fullpath'] = os.path.join(builddir, relpath)
|
||||
basename = os.path.basename(relpath)
|
||||
rpmdata[basename] = rpminfo
|
||||
rpmdata[bnp] = rpminfo
|
||||
|
||||
# sanity check
|
||||
for fn in rpmdata:
|
||||
if fn not in pkglist:
|
||||
raise koji.GenericError("No signature data for: %s" % fn)
|
||||
for fn in pkglist:
|
||||
if fn not in rpmdata:
|
||||
raise koji.GenericError("RPM missing from pkglist: %s" % fn)
|
||||
# move the uploaded files
|
||||
dirnames = set([os.path.dirname(fm[1]) for fm in file_moves])
|
||||
for dirname in dirnames:
|
||||
koji.ensuredir(dirname)
|
||||
for src, dst in file_moves:
|
||||
safer_move(src, dst)
|
||||
|
||||
# hardlink or copy the rpms into the final repodir
|
||||
# TODO: properly consider split-volume functionality
|
||||
for fn in rpmdata:
|
||||
# hardlink or copy the rpms into the final repodir
|
||||
# TODO: properly consider split-volume functionality
|
||||
rpminfo = rpmdata[fn]
|
||||
rpmpath = rpminfo['_fullpath']
|
||||
bnp = fn
|
||||
bnplet = bnp[0].lower()
|
||||
koji.ensuredir(os.path.join(archdir, bnplet))
|
||||
l_dst = os.path.join(archdir, bnplet, bnp)
|
||||
ddir = os.path.join(archdir, 'Packages', bnplet)
|
||||
koji.ensuredir(ddir)
|
||||
l_dst = os.path.join(ddir, bnp)
|
||||
if os.path.exists(l_dst):
|
||||
raise koji.GenericError("File already in repo: %s", l_dst)
|
||||
logger.debug("os.link(%r, %r)", rpmpath, l_dst)
|
||||
|
|
@ -12717,8 +12745,7 @@ class HostExports(object):
|
|||
os.link(rpmpath, l_dst)
|
||||
except OSError as ose:
|
||||
if ose.errno == 18:
|
||||
shutil.copy2(
|
||||
rpmpath, os.path.join(archdir, bnplet, bnp))
|
||||
shutil.copy2(rpmpath, l_dst)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
|
|
|||
|
|
@ -252,14 +252,16 @@ Options:
|
|||
--allow-missing-signatures
|
||||
For RPMs not signed with a desired key, fall back to
|
||||
the primary copy
|
||||
--arch=ARCH Indicate an architecture to consider. The default is
|
||||
-a ARCH, --arch=ARCH Indicate an architecture to consider. The default is
|
||||
all architectures associated with the given tag. This
|
||||
option may be specified multiple times.
|
||||
--with-src Also generate a src repo
|
||||
--split-debuginfo Split debuginfo info a separate repo for each arch
|
||||
--comps=COMPS Include a comps file in the repodata
|
||||
--delta-rpms=REPO Create delta rpms. REPO can be the id of another dist
|
||||
repo or the name of a tag that has a dist repo. May be
|
||||
specified multiple times.
|
||||
--event=EVENT create a dist repository based on a Brew event
|
||||
--event=EVENT Use tag content at event
|
||||
--non-latest Include older builds, not just the latest
|
||||
--multilib=CONFIG Include multilib packages in the repository using the
|
||||
given config file
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
|
||||
import unittest
|
||||
import json
|
||||
import mock
|
||||
import os
|
||||
import shutil
|
||||
|
|
@ -106,7 +107,7 @@ class TestDistRepoMove(unittest.TestCase):
|
|||
'create_ts': 1487256924.72718,
|
||||
'creation_time': '2017-02-16 14:55:24.727181',
|
||||
'id': 47,
|
||||
'state': 1,
|
||||
'state': 0, # INIT
|
||||
'tag_id': 2,
|
||||
'tag_name': 'my-tag'}
|
||||
self.arch = 'x86_64'
|
||||
|
|
@ -123,19 +124,18 @@ class TestDistRepoMove(unittest.TestCase):
|
|||
os.makedirs(uploaddir)
|
||||
|
||||
# place some test files
|
||||
self.files = ['foo.drpm', 'repomd.xml']
|
||||
self.files = ['drpms/foo.drpm', 'repodata/repomd.xml']
|
||||
self.expected = ['x86_64/drpms/foo.drpm', 'x86_64/repodata/repomd.xml']
|
||||
for fn in self.files:
|
||||
path = os.path.join(uploaddir, fn)
|
||||
koji.ensuredir(os.path.dirname(path))
|
||||
with open(path, 'w') as fo:
|
||||
fo.write('%s' % fn)
|
||||
fo.write('%s' % os.path.basename(fn))
|
||||
|
||||
# generate pkglist file and sigmap
|
||||
# generate pkglist file
|
||||
self.files.append('pkglist')
|
||||
plist = os.path.join(uploaddir, 'pkglist')
|
||||
nvrs = ['aaa-1.0-2', 'bbb-3.0-5', 'ccc-8.0-13','ddd-21.0-34']
|
||||
self.sigmap = []
|
||||
self.rpms = {}
|
||||
self.builds ={}
|
||||
self.key = '4c8da725'
|
||||
|
|
@ -153,15 +153,30 @@ class TestDistRepoMove(unittest.TestCase):
|
|||
fo.write('%s' % basename)
|
||||
f_pkglist.write(path)
|
||||
f_pkglist.write('\n')
|
||||
self.expected.append('x86_64/%s/%s' % (basename[0], basename))
|
||||
self.expected.append('x86_64/Packages/%s/%s' % (basename[0], basename))
|
||||
build_id = len(self.builds) + 10000
|
||||
rpm_id = len(self.rpms) + 20000
|
||||
binfo['id'] = build_id
|
||||
rpminfo['build_id'] = build_id
|
||||
rpminfo['id'] = rpm_id
|
||||
rpminfo['sigkey'] = self.key
|
||||
rpminfo['size'] = 1024
|
||||
rpminfo['payloadhash'] = 'helloworld'
|
||||
self.builds[build_id] = binfo
|
||||
self.rpms[rpm_id] = rpminfo
|
||||
self.sigmap.append([rpm_id, self.key])
|
||||
|
||||
# write kojipkgs
|
||||
kojipkgs = {}
|
||||
for rpminfo in self.rpms.values():
|
||||
bnp = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % rpminfo
|
||||
kojipkgs[bnp] = rpminfo
|
||||
with open("%s/kojipkgs" % uploaddir, "w") as fp:
|
||||
json.dump(kojipkgs, fp, indent=4)
|
||||
self.files.append('kojipkgs')
|
||||
|
||||
# write manifest
|
||||
with open("%s/repo_manifest" % uploaddir, "w") as fp:
|
||||
json.dump(self.files, fp, indent=4)
|
||||
|
||||
# mocks
|
||||
self.repo_info = mock.patch('kojihub.repo_info').start()
|
||||
|
|
@ -187,8 +202,7 @@ class TestDistRepoMove(unittest.TestCase):
|
|||
|
||||
def test_distRepoMove(self):
|
||||
exports = kojihub.HostExports()
|
||||
exports.distRepoMove(self.rinfo['id'], self.uploadpath,
|
||||
list(self.files), self.arch, self.sigmap)
|
||||
exports.distRepoMove(self.rinfo['id'], self.uploadpath, self.arch)
|
||||
# check result
|
||||
repodir = self.topdir + '/repos-dist/%(tag_name)s/%(id)s' % self.rinfo
|
||||
for relpath in self.expected:
|
||||
|
|
|
|||
75
tests/test_hub/test_repos.py
Normal file
75
tests/test_hub/test_repos.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import mock
|
||||
import unittest
|
||||
|
||||
import koji
|
||||
import kojihub
|
||||
|
||||
|
||||
QP = kojihub.QueryProcessor
|
||||
IP = kojihub.InsertProcessor
|
||||
UP = kojihub.UpdateProcessor
|
||||
|
||||
|
||||
class TestRepoFunctions(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.QueryProcessor = mock.patch('kojihub.QueryProcessor',
|
||||
side_effect=self.getQuery).start()
|
||||
self.queries = []
|
||||
self.InsertProcessor = mock.patch('kojihub.InsertProcessor',
|
||||
side_effect=self.getInsert).start()
|
||||
self.inserts = []
|
||||
self.UpdateProcessor = mock.patch('kojihub.UpdateProcessor',
|
||||
side_effect=self.getUpdate).start()
|
||||
self.updates = []
|
||||
self._dml = mock.patch('kojihub._dml').start()
|
||||
|
||||
def tearDown(self):
|
||||
mock.patch.stopall()
|
||||
|
||||
def getQuery(self, *args, **kwargs):
|
||||
query = QP(*args, **kwargs)
|
||||
query.execute = mock.MagicMock()
|
||||
self.queries.append(query)
|
||||
return query
|
||||
|
||||
def getInsert(self, *args, **kwargs):
|
||||
insert = IP(*args, **kwargs)
|
||||
insert.execute = mock.MagicMock()
|
||||
self.inserts.append(insert)
|
||||
return insert
|
||||
|
||||
def getUpdate(self, *args, **kwargs):
|
||||
update = UP(*args, **kwargs)
|
||||
update.execute = mock.MagicMock()
|
||||
self.updates.append(update)
|
||||
return update
|
||||
|
||||
def test_repo_expire_older(self):
|
||||
kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id)
|
||||
self.assertEqual(len(self.updates), 1)
|
||||
update = self.updates[0]
|
||||
self.assertEqual(update.table, 'repo')
|
||||
self.assertEqual(update.data, {'state': koji.REPO_EXPIRED})
|
||||
self.assertEqual(update.rawdata, {})
|
||||
self.assertEqual(update.values['event_id'], mock.sentinel.event_id)
|
||||
self.assertEqual(update.values['tag_id'], mock.sentinel.tag_id)
|
||||
self.assertEqual(update.values['dist'], None)
|
||||
if 'dist = %(dist)s' in update.clauses:
|
||||
raise Exception('Unexpected dist condition')
|
||||
|
||||
# and with dist specified
|
||||
for dist in True, False:
|
||||
self.updates = []
|
||||
kojihub.repo_expire_older(mock.sentinel.tag_id, mock.sentinel.event_id,
|
||||
dist=dist)
|
||||
self.assertEqual(len(self.updates), 1)
|
||||
update = self.updates[0]
|
||||
self.assertEqual(update.table, 'repo')
|
||||
self.assertEqual(update.data, {'state': koji.REPO_EXPIRED})
|
||||
self.assertEqual(update.rawdata, {})
|
||||
self.assertEqual(update.values['event_id'], mock.sentinel.event_id)
|
||||
self.assertEqual(update.values['tag_id'], mock.sentinel.tag_id)
|
||||
self.assertEqual(update.values['dist'], dist)
|
||||
if 'dist = %(dist)s' not in update.clauses:
|
||||
raise Exception('Missing dist condition')
|
||||
|
|
@ -225,7 +225,7 @@ $value
|
|||
#end if
|
||||
#elif $task.method == 'distRepo'
|
||||
<strong>Tag:</strong> <a href="taginfo?tagID=$tag.id">$tag.name</a><br/>
|
||||
<strong>Repo ID:</strong> $params[1]<br/>
|
||||
<strong>Repo ID:</strong> <a href="$pathinfo.distrepo($params[1],$tag.name)">$params[1]</a></br>
|
||||
<strong>Keys:</strong> $printValue(0, $params[2])<br/>
|
||||
$printOpts($params[3])
|
||||
#elif $task.method == 'prepRepo'
|
||||
|
|
@ -243,7 +243,7 @@ $value
|
|||
#end if
|
||||
#elif $task.method == 'createdistrepo'
|
||||
<strong>Tag:</strong> <a href="taginfo?tagID=$tag.id">$tag.name</a><br/>
|
||||
<strong>Repo ID:</strong> $params[1]<br/>
|
||||
<strong>Repo ID:</strong> <a href="$pathinfo.distrepo($params[1],$tag.name)">$params[1]</a></br>
|
||||
<strong>Arch:</strong> $printValue(0, $params[2])<br/>
|
||||
<strong>Keys:</strong> $printValue(0, $params[3])<br/>
|
||||
<strong>Options:</strong> $printMap($params[4], ' ')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue