support for split storage
This commit is contained in:
Mike McLean
parent
919716b8be
commit
73d44e199a
12 changed files with 272 additions and 103 deletions
|
|
@ -52,7 +52,7 @@ import Cheetah.Template
|
|||
from ConfigParser import ConfigParser
|
||||
from fnmatch import fnmatch
|
||||
from gzip import GzipFile
|
||||
from optparse import OptionParser
|
||||
from optparse import OptionParser, SUPPRESS_HELP
|
||||
from StringIO import StringIO
|
||||
|
||||
#imports for LiveCD and Appliance handler
|
||||
|
|
@ -221,20 +221,8 @@ class BuildRoot(object):
|
|||
repo_id = self.repoid
|
||||
tag_name = self.tag_name
|
||||
|
||||
topurl = None
|
||||
topdir = None
|
||||
if hasattr(self.options, 'topurl'):
|
||||
topurl = self.options.topurl
|
||||
if hasattr(self.options, 'topdir'):
|
||||
topdir = self.options.topdir
|
||||
if topurl:
|
||||
pi = koji.PathInfo(topdir=topurl)
|
||||
repourl = pi.repo(repo_id, tag_name) + '/maven2'
|
||||
elif topdir:
|
||||
pi = koji.PathInfo(topdir=topdir)
|
||||
repourl = 'file://' + pi.repo(repo_id, tag_name) + '/maven2'
|
||||
else:
|
||||
raise koji.BuildError, 'either topurl or topdir must be specified in the config file'
|
||||
pi = koji.PathInfo(topdir=self.options.topurl)
|
||||
repourl = pi.repo(repo_id, tag_name) + '/maven2'
|
||||
localrepo = localrepodir[len(self.rootdir()):]
|
||||
|
||||
settings = """<settings xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
|
|
@ -1745,10 +1733,7 @@ class ImageTask(BaseTaskHandler):
|
|||
ks.handler.repo.repoList.append(repo_class(baseurl=user_repo, name='koji-override-%i' % index))
|
||||
index += 1
|
||||
else:
|
||||
topurl = getattr(self.options, 'topurl')
|
||||
if not topurl:
|
||||
raise koji.LiveCDError, 'topurl must be defined in kojid.conf'
|
||||
path_info = koji.PathInfo(topdir=topurl)
|
||||
path_info = koji.PathInfo(topdir=self.options.topurl)
|
||||
repopath = path_info.repo(repo_info['id'],
|
||||
target_info['build_tag_name'])
|
||||
baseurl = '%s/%s' % (repopath, arch)
|
||||
|
|
@ -2426,7 +2411,9 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r
|
|||
output += "\r\n\r\n"
|
||||
else:
|
||||
output = ''
|
||||
|
||||
|
||||
pathinfo = koji.PathInfo(topdir=self.options.topurl)
|
||||
buildurl = pathinfo.build(build)
|
||||
# list states here to make them go in the correct order
|
||||
for task_state in ['failed', 'canceled', 'closed']:
|
||||
if tasks[task_state]:
|
||||
|
|
@ -2444,12 +2431,12 @@ Build Info: %(weburl)s/buildinfo?buildID=%(build_id)i\r
|
|||
output += "%s:\r\n" % filetype
|
||||
for file in task[filetype]:
|
||||
if filetype == 'rpms':
|
||||
output += " %s\r\n" % '/'.join([self.options.pkgurl, build['name'], build['version'], build['release'], task['build_arch'], file])
|
||||
output += " %s\r\n" % '/'.join([buildurl, task['build_arch'], file])
|
||||
elif filetype == 'logs':
|
||||
if tasks[task_state] != 'closed':
|
||||
output += " %s/getfile?taskID=%s&name=%s\r\n" % (weburl, task['id'], file)
|
||||
else:
|
||||
output += " %s\r\n" % '/'.join([self.options.pkgurl, build['name'], build['version'], build['release'], 'data', 'logs', task['build_arch'], file])
|
||||
output += " %s\r\n" % '/'.join([buildurl, 'data', 'logs', task['build_arch'], file])
|
||||
elif task[filetype] == 'misc':
|
||||
output += " %s/getfile?taskID=%s&name=%s\r\n" % (weburl, task['id'], file)
|
||||
output += "\r\n"
|
||||
|
|
@ -2575,7 +2562,7 @@ class CreaterepoTask(BaseTaskHandler):
|
|||
|
||||
def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):
|
||||
koji.ensuredir(self.outdir)
|
||||
cmd = ['/usr/bin/createrepo', '-vd', '-o', self.outdir, '-u', self.options.pkgurl]
|
||||
cmd = ['/usr/bin/createrepo', '-vd', '-o', self.outdir, '-u', self.options.topurl]
|
||||
if pkglist is not None:
|
||||
cmd.extend(['-i', pkglist])
|
||||
if os.path.isfile(groupdata):
|
||||
|
|
@ -2602,8 +2589,7 @@ class CreaterepoTask(BaseTaskHandler):
|
|||
if pkglist is None:
|
||||
cmd.append(self.outdir)
|
||||
else:
|
||||
pkgdir = os.path.join(self.pathinfo.topdir, 'packages/')
|
||||
cmd.append(pkgdir)
|
||||
cmd.append(self.options.topdir)
|
||||
|
||||
logfile = '%s/createrepo.log' % self.workdir
|
||||
status = log_output(self.session, cmd[0], cmd, logfile, self.getUploadDir(), logerror=True)
|
||||
|
|
@ -2713,6 +2699,7 @@ class WaitrepoTask(BaseTaskHandler):
|
|||
def get_options():
|
||||
"""process options from command line and config file"""
|
||||
# parse command line args
|
||||
logger = logging.getLogger("koji.build")
|
||||
parser = OptionParser()
|
||||
parser.add_option("-c", "--config", dest="configFile",
|
||||
help="use alternate configuration file", metavar="FILE",
|
||||
|
|
@ -2748,7 +2735,7 @@ def get_options():
|
|||
parser.add_option("--mockdir", help="Specify mockdir")
|
||||
parser.add_option("--mockuser", help="User to run mock as")
|
||||
parser.add_option("-s", "--server", help="url of XMLRPC server")
|
||||
parser.add_option("--pkgurl", help="url of packages directory")
|
||||
parser.add_option("--pkgurl", help=SUPPRESS_HELP)
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
if args:
|
||||
|
|
@ -2829,10 +2816,17 @@ def get_options():
|
|||
koji.ensuredir(options.workdir)
|
||||
|
||||
if not options.server:
|
||||
parser.error("--server argument required")
|
||||
msg = "the server option is required"
|
||||
logger.error(msg)
|
||||
parser.error(msg)
|
||||
|
||||
if not options.pkgurl:
|
||||
parser.error("--pkgurl argument required")
|
||||
if not options.topurl:
|
||||
msg = "the topurl option is required"
|
||||
logger.error(msg)
|
||||
parser.error(msg)
|
||||
|
||||
if options.pkgurl:
|
||||
logger.warning("The pkgurl option is obsolete")
|
||||
|
||||
return options
|
||||
|
||||
|
|
|
|||
|
|
@ -35,8 +35,8 @@
|
|||
; The URL for the xmlrpc server
|
||||
server=http://hub.example.com/kojihub
|
||||
|
||||
; The URL for the packages tree
|
||||
pkgurl=http://hub.example.com/packages
|
||||
; The URL for the file access
|
||||
topurl=http://hub.example.com/kojifiles
|
||||
|
||||
; A space-separated list of hostname:repository[:use_common] tuples that kojid is authorized to checkout from (no quotes).
|
||||
; Wildcards (as supported by fnmatch) are allowed.
|
||||
|
|
|
|||
46
cli/koji
46
cli/koji
|
|
@ -118,7 +118,8 @@ def get_options():
|
|||
parser.add_option("-s", "--server", help=_("url of XMLRPC server"))
|
||||
parser.add_option("--topdir", help=_("specify topdir"))
|
||||
parser.add_option("--weburl", help=_("url of the Koji web interface"))
|
||||
parser.add_option("--pkgurl", help=_("url of the Koji package tree"))
|
||||
parser.add_option("--topurl", help=_("url for Koji file access"))
|
||||
parser.add_option("--pkgurl", help=optparse.SUPPRESS_HELP)
|
||||
parser.add_option("--help-commands", action="store_true", default=False, help=_("list commands"))
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
|
|
@ -151,7 +152,8 @@ def get_options():
|
|||
defaults = {
|
||||
'server' : 'http://localhost/kojihub',
|
||||
'weburl' : 'http://localhost/koji',
|
||||
'pkgurl' : 'http://localhost/packages',
|
||||
'topurl' : None,
|
||||
'pkgurl' : None,
|
||||
'topdir' : '/mnt/koji',
|
||||
'max_retries' : None,
|
||||
'retry_interval': None,
|
||||
|
|
@ -205,6 +207,18 @@ def get_options():
|
|||
koji.BASEDIR = options.topdir
|
||||
koji.pathinfo.topdir = options.topdir
|
||||
|
||||
#pkgurl is obsolete
|
||||
if options.pkgurl:
|
||||
if options.topurl:
|
||||
print "Warning: the pkgurl option is obsolete"
|
||||
else:
|
||||
suggest = re.sub(r'/packages/?$', '', options.pkgurl)
|
||||
if suggest != options.pkgurl:
|
||||
print "Warning: the pkgurl option is obsolete, using topurl=%r" % suggest
|
||||
options.topurl = suggest
|
||||
else:
|
||||
print "Warning: The pkgurl option is obsolete, please use topurl instead"
|
||||
|
||||
return options, cmd, args[1:]
|
||||
|
||||
def ensure_connection(session):
|
||||
|
|
@ -5311,7 +5325,7 @@ def anon_handle_download_build(options, session, args):
|
|||
parser.add_option("--latestfrom", dest="latestfrom", help=_("Download the latest build from this tag"))
|
||||
parser.add_option("--debuginfo", action="store_true", help=_("Also download -debuginfo rpms"))
|
||||
parser.add_option("--key", help=_("Download rpms signed with the given key"))
|
||||
parser.add_option("--topurl", metavar="URL",
|
||||
parser.add_option("--topurl", metavar="URL", default=options.topurl,
|
||||
help=_("URL under which Koji files are accessible"))
|
||||
parser.add_option("-q", "--quiet", action="store_true", help=_("Do not display progress meter"),
|
||||
default=options.quiet)
|
||||
|
|
@ -5350,6 +5364,12 @@ def anon_handle_download_build(options, session, args):
|
|||
print "No such build: %s" % build
|
||||
return 1
|
||||
|
||||
if not suboptions.topurl:
|
||||
print "You must specify --topurl to download files"
|
||||
return 1
|
||||
pathinfo = koji.PathInfo(topdir=suboptions.topurl)
|
||||
build_url = pathinfo.build(info)
|
||||
|
||||
urls = []
|
||||
if suboptions.type:
|
||||
archives = session.listArchives(buildID=info['id'], type=suboptions.type)
|
||||
|
|
@ -5357,18 +5377,13 @@ def anon_handle_download_build(options, session, args):
|
|||
print "No %s archives available for %s" % (suboptions.type, koji.buildLabel(info))
|
||||
return 1
|
||||
if suboptions.type == 'maven':
|
||||
if not suboptions.topurl:
|
||||
print "You must specify --topurl to download Maven archives"
|
||||
return 1
|
||||
maven_pi = koji.PathInfo(topdir=suboptions.topurl)
|
||||
maven_info = session.getMavenBuild(info['id'], strict=True)
|
||||
for archive in archives:
|
||||
urls.append((maven_pi.mavenbuild(info, maven_info) + '/' + archive['filename'], archive['filename']))
|
||||
urls.append((pathinfo.mavenbuild(info, maven_info) + '/' + archive['filename'], archive['filename']))
|
||||
elif suboptions.type == 'win':
|
||||
for archive in archives:
|
||||
url = '%s/%s/%s/%s/win/%s' % (options.pkgurl, info['name'], info['version'], info['release'],
|
||||
koji.pathinfo.winfile(archive))
|
||||
urls.append((url, koji.pathinfo.winfile(archive)))
|
||||
url = '%s/win/%s' % (build_url, pathinfo.winfile(archive))
|
||||
urls.append((url, pathinfo.winfile(archive)))
|
||||
else:
|
||||
# can't happen
|
||||
assert False
|
||||
|
|
@ -5387,11 +5402,10 @@ def anon_handle_download_build(options, session, args):
|
|||
if not suboptions.debuginfo and koji.is_debuginfo(rpm['name']):
|
||||
continue
|
||||
if suboptions.key:
|
||||
fname = koji.pathinfo.signed(rpm, suboptions.key)
|
||||
fname = pathinfo.signed(rpm, suboptions.key)
|
||||
else:
|
||||
fname = koji.pathinfo.rpm(rpm)
|
||||
url = '%s/%s/%s/%s/%s' % (options.pkgurl, info['name'], info['version'], info['release'],
|
||||
fname)
|
||||
fname = pathinfo.rpm(rpm)
|
||||
url = '%s/%s' % (build_url, fname)
|
||||
urls.append((url, os.path.basename(fname)))
|
||||
|
||||
if suboptions.quiet:
|
||||
|
|
@ -5533,7 +5547,7 @@ def anon_handle_search(options, session, args):
|
|||
usage = _("usage: %prog search [options] search_type pattern")
|
||||
usage += _('\nAvailable search types: %s') % ', '.join(_search_types)
|
||||
usage += _("\n(Specify the --help global option for a list of other help options)")
|
||||
parser = optparse.OptionParser(usage=usage)
|
||||
parser = OptionParser(usage=usage)
|
||||
parser.add_option("-r", "--regex", action="store_true", help=_("treat pattern as regex"))
|
||||
parser.add_option("--exact", action="store_true", help=_("exact matches only"))
|
||||
main_options = options
|
||||
|
|
|
|||
14
docs/schema-upgrade-1.5-1.7.sql
Normal file
14
docs/schema-upgrade-1.5-1.7.sql
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
BEGIN;
|
||||
|
||||
CREATE TABLE volume (
|
||||
id SERIAL NOT NULL PRIMARY KEY,
|
||||
name TEXT UNIQUE NOT NULL
|
||||
) WITHOUT OIDS;
|
||||
|
||||
INSERT INTO volume (id, name) VALUES (0, 'DEFAULT');
|
||||
|
||||
ALTER TABLE build ADD COLUMN volume_id INTEGER REFERENCES volume (id);
|
||||
UPDATE build SET volume_id = 0;
|
||||
ALTER TABLE build ALTER COLUMN volume_id SET NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -261,6 +261,13 @@ CREATE TABLE package (
|
|||
-- (implicitly created by unique constraint)
|
||||
|
||||
|
||||
CREATE TABLE volume (
|
||||
id SERIAL NOT NULL PRIMARY KEY,
|
||||
name TEXT UNIQUE NOT NULL
|
||||
) WITHOUT OIDS;
|
||||
|
||||
INSERT INTO volume (id, name) VALUES (0, 'DEFAULT');
|
||||
|
||||
-- here we track the built packages
|
||||
-- this is at the srpm level, since builds are by srpm
|
||||
-- see rpminfo for isolated packages
|
||||
|
|
@ -269,6 +276,7 @@ CREATE TABLE package (
|
|||
-- null, or may point to a deleted task.
|
||||
CREATE TABLE build (
|
||||
id SERIAL NOT NULL PRIMARY KEY,
|
||||
volume_id INTEGER NOT NULL REFERENCES volume (id),
|
||||
pkg_id INTEGER NOT NULL REFERENCES package (id) DEFERRABLE,
|
||||
version TEXT NOT NULL,
|
||||
release TEXT NOT NULL,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
#
|
||||
|
||||
Alias /kojihub "/usr/share/koji-hub/XMLRPC"
|
||||
Alias /kojifiles "/mnt/koji/"
|
||||
|
||||
<Directory /usr/share/koji-hub>
|
||||
SetHandler mod_python
|
||||
|
|
@ -15,6 +16,13 @@ Alias /kojihub "/usr/share/koji-hub/XMLRPC"
|
|||
PythonAutoReload Off
|
||||
</Directory>
|
||||
|
||||
<Directory "/mnt/koji">
|
||||
Options Indexes
|
||||
AllowOverride None
|
||||
Order allow,deny
|
||||
Allow from all
|
||||
</Directory>
|
||||
|
||||
# uncomment this to enable authentication via SSL client certificates
|
||||
# <Location /kojihub/ssllogin>
|
||||
# SSLVerifyClient require
|
||||
|
|
|
|||
159
hub/kojihub.py
159
hub/kojihub.py
|
|
@ -40,6 +40,7 @@ import os
|
|||
import random
|
||||
import re
|
||||
import rpm
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
|
@ -1087,6 +1088,7 @@ def readTaggedBuilds(tag,event=None,inherit=False,latest=False,package=None,owne
|
|||
('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'),
|
||||
('build.task_id','task_id'),
|
||||
('events.id', 'creation_event_id'), ('events.time', 'creation_time'),
|
||||
('volume.id', 'volume_id'), ('volume.name', 'volume_name'),
|
||||
('package.id', 'package_id'), ('package.name', 'package_name'),
|
||||
('package.name', 'name'),
|
||||
("package.name || '-' || build.version || '-' || build.release", 'nvr'),
|
||||
|
|
@ -1115,6 +1117,7 @@ def readTaggedBuilds(tag,event=None,inherit=False,latest=False,package=None,owne
|
|||
JOIN users ON users.id = build.owner
|
||||
JOIN events ON events.id = build.create_event
|
||||
JOIN package ON package.id = build.pkg_id
|
||||
JOIN volume ON volume.id = build.volume_id
|
||||
WHERE %s AND tag_id=%%(tagid)s
|
||||
AND build.state=%%(st_complete)i
|
||||
""" % (', '.join([pair[0] for pair in fields]), type_join, eventCondition(event, 'tag_listing'))
|
||||
|
|
@ -2080,6 +2083,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
|
|||
packages = {}
|
||||
for repoarch in repo_arches:
|
||||
packages.setdefault(repoarch, [])
|
||||
relpathinfo = koji.PathInfo(topdir='')
|
||||
for rpminfo in rpms:
|
||||
if not with_debuginfo and koji.is_debuginfo(rpminfo['name']):
|
||||
continue
|
||||
|
|
@ -2094,7 +2098,8 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
|
|||
# Do not create a repo for arches not in the arch list for this tag
|
||||
continue
|
||||
build = builds[rpminfo['build_id']]
|
||||
rpminfo['path'] = "%s/%s" % (koji.pathinfo.build(build), koji.pathinfo.rpm(rpminfo))
|
||||
rpminfo['relpath'] = "%s/%s" % (relpathinfo.build(build), relpathinfo.rpm(rpminfo))
|
||||
rpminfo['relpath'] = rpminfo['relpath'].lstrip('/')
|
||||
packages.setdefault(repoarch,[]).append(rpminfo)
|
||||
#generate comps and groups.spec
|
||||
groupsdir = "%s/groups" % (repodir)
|
||||
|
|
@ -2107,7 +2112,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
|
|||
if context.opts.get('EnableMaven') and tinfo['maven_support']:
|
||||
maven_builds = maven_tag_packages(tinfo, event_id)
|
||||
|
||||
#link packages
|
||||
#generate pkglist files
|
||||
for arch in packages.iterkeys():
|
||||
if arch in ['src','noarch']:
|
||||
continue
|
||||
|
|
@ -2117,16 +2122,16 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None):
|
|||
pkglist = file(os.path.join(repodir, arch, 'pkglist'), 'w')
|
||||
logger.info("Creating package list for %s" % arch)
|
||||
for rpminfo in packages[arch]:
|
||||
pkglist.write(rpminfo['path'].split(os.path.join(koji.pathinfo.topdir, 'packages/'))[1] + '\n')
|
||||
pkglist.write(rpminfo['relpath'] + '\n')
|
||||
#noarch packages
|
||||
for rpminfo in packages.get('noarch',[]):
|
||||
pkglist.write(rpminfo['path'].split(os.path.join(koji.pathinfo.topdir, 'packages/'))[1] + '\n')
|
||||
pkglist.write(rpminfo['relpath'] + '\n')
|
||||
# srpms
|
||||
if with_src:
|
||||
srpmdir = "%s/%s" % (repodir,'src')
|
||||
koji.ensuredir(srpmdir)
|
||||
for rpminfo in packages.get('src',[]):
|
||||
pkglist.write(rpminfo['path'].split(os.path.join(koji.pathinfo.topdir, 'packages/'))[1] + '\n')
|
||||
pkglist.write(rpminfo['relpath'] + '\n')
|
||||
pkglist.close()
|
||||
#write list of blocked packages
|
||||
blocklist = file(os.path.join(repodir, arch, 'blocklist'), 'w')
|
||||
|
|
@ -3066,6 +3071,8 @@ def get_build(buildInfo, strict=False):
|
|||
task_id: ID of the task that kicked off the build
|
||||
owner_id: ID of the user who kicked off the build
|
||||
owner_name: name of the user who kicked off the build
|
||||
volume_id: ID of the storage volume
|
||||
volume_name: name of the storage volume
|
||||
creation_event_id: id of the create_event
|
||||
creation_time: time the build was created (text)
|
||||
creation_ts: time the build was created (epoch)
|
||||
|
|
@ -3086,6 +3093,7 @@ def get_build(buildInfo, strict=False):
|
|||
('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'),
|
||||
('build.task_id', 'task_id'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'),
|
||||
('package.id', 'package_id'), ('package.name', 'package_name'), ('package.name', 'name'),
|
||||
('volume.id', 'volume_id'), ('volume.name', 'volume_name'),
|
||||
("package.name || '-' || build.version || '-' || build.release", 'nvr'),
|
||||
('EXTRACT(EPOCH FROM events.time)','creation_ts'),
|
||||
('EXTRACT(EPOCH FROM build.completion_time)','completion_ts'),
|
||||
|
|
@ -3094,6 +3102,7 @@ def get_build(buildInfo, strict=False):
|
|||
FROM build
|
||||
JOIN events ON build.create_event = events.id
|
||||
JOIN package on build.pkg_id = package.id
|
||||
JOIN volume on build.volume_id = volume.id
|
||||
JOIN users on build.owner = users.id
|
||||
WHERE build.id = %%(buildID)i""" % ', '.join([pair[0] for pair in fields])
|
||||
|
||||
|
|
@ -3891,6 +3900,77 @@ def new_package(name,strict=True):
|
|||
c.execute(q,locals())
|
||||
return pkg_id
|
||||
|
||||
|
||||
def add_volume(name, strict=True):
|
||||
"""Add a new storage volume in the database"""
|
||||
context.session.assertPerm('admin')
|
||||
if strict:
|
||||
volinfo = lookup_name('volume', name, strict=False)
|
||||
if volinfo:
|
||||
raise koji.GenericError, 'volume %s already exists' % name
|
||||
volinfo = lookup_name('volume', name, strict=False, create=True)
|
||||
|
||||
def remove_volume(volume):
|
||||
"""Remove unused storage volume from the database"""
|
||||
context.session.assertPerm('admin')
|
||||
volinfo = lookup_name('volume', volume, strict=True)
|
||||
query = QueryProcessor(tables=['build'], clauses=['volume_id=%(id)i'],
|
||||
values=volinfo, columns=['id'], opts={'limit':1})
|
||||
if query.execute():
|
||||
raise koji.GenericError, 'volume %(name)s has build references' % volinfo
|
||||
delete = """DELETE FROM volume WHERE id=%(id)i"""
|
||||
_dml(delete, volinfo)
|
||||
|
||||
def list_volumes():
|
||||
"""List storage volumes"""
|
||||
return QueryProcessor(tables=['volume'], columns=['id', 'name']).execute()
|
||||
|
||||
def change_build_volume(build, volume, strict=True):
|
||||
"""Move a build to a different storage volume"""
|
||||
context.session.assertPerm('admin')
|
||||
volinfo = lookup_name('volume', volume, strict=True)
|
||||
binfo = get_build(build, strict=True)
|
||||
if binfo['volume_id'] == volinfo['id']:
|
||||
if strict:
|
||||
raise koji.GenericError, "Build %(nvr)s already on volume %(volume_name)s" % binfo
|
||||
else:
|
||||
#nothing to do
|
||||
return
|
||||
state = koji.BUILD_STATES[binfo['state']]
|
||||
if state not in ['COMPLETE', 'DELETED']:
|
||||
raise koji.GenericError, "Build %s is %s" % (binfo['nvr'], state)
|
||||
|
||||
# First copy the build dir(s)
|
||||
dir_moves = []
|
||||
old_binfo = binfo.copy()
|
||||
binfo['volume_id'] = volinfo['id']
|
||||
binfo['volume_name'] = volinfo['name']
|
||||
olddir = koji.pathinfo.build(old_binfo)
|
||||
if os.path.exists(olddir):
|
||||
newdir = koji.pathinfo.build(binfo)
|
||||
dir_moves.append([olddir, newdir])
|
||||
maven_info = get_maven_build(binfo['id'])
|
||||
if maven_info:
|
||||
olddir = koji.pathinfo.mavenbuild(old_binfo, maven_info)
|
||||
if os.path.exists(olddir):
|
||||
newdir = koji.pathinfo.mavenbuild(binfo, maven_info)
|
||||
dir_moves.append([olddir, newdir])
|
||||
for olddir, newdir in dir_moves:
|
||||
shutil.copytree(olddir, newdir, symlinks=True)
|
||||
|
||||
# Second, update the db
|
||||
koji.plugin.run_callbacks('preBuildStateChange', attribute='volume_id', old=old_binfo['volume_id'], new=volinfo['id'], info=binfo)
|
||||
update = UpdateProcessor('build', clauses=['id=%(id)i'], values=binfo)
|
||||
update.set(volume_id=volinfo['id'])
|
||||
update.execute()
|
||||
|
||||
# Third, delete the old content
|
||||
for olddir, newdir in dir_moves:
|
||||
koji.util.rmtree(olddir)
|
||||
|
||||
koji.plugin.run_callbacks('postBuildStateChange', attribute='volume_id', old=old_binfo['volume_id'], new=volinfo['id'], info=binfo)
|
||||
|
||||
|
||||
def new_build(data):
|
||||
"""insert a new build entry"""
|
||||
data = data.copy()
|
||||
|
|
@ -3908,6 +3988,7 @@ def new_build(data):
|
|||
data.setdefault('completion_time', 'NOW')
|
||||
data.setdefault('owner',context.session.user_id)
|
||||
data.setdefault('task_id',None)
|
||||
data.setdefault('volume_id', 0)
|
||||
#check for existing build
|
||||
# TODO - table lock?
|
||||
q="""SELECT id,state,task_id FROM build
|
||||
|
|
@ -3939,14 +4020,11 @@ def new_build(data):
|
|||
else:
|
||||
koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, new=data['state'], info=data)
|
||||
#insert the new data
|
||||
data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'state', 'volume_id',
|
||||
'task_id', 'owner', 'completion_time'])
|
||||
data['id'] = _singleValue("SELECT nextval('build_id_seq')")
|
||||
q="""
|
||||
INSERT INTO build (id,pkg_id,version,release,epoch,state,
|
||||
task_id,owner,completion_time)
|
||||
VALUES (%(id)i,%(pkg_id)i,%(version)s,%(release)s,%(epoch)s,
|
||||
%(state)s,%(task_id)s,%(owner)s,%(completion_time)s)
|
||||
"""
|
||||
_dml(q, data)
|
||||
insert = InsertProcessor('build', data=data)
|
||||
insert.execute()
|
||||
koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=None, new=data['state'], info=data)
|
||||
#return build_id
|
||||
return data['id']
|
||||
|
|
@ -4037,12 +4115,11 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None)
|
|||
WHERE id=%(build_id)i"""
|
||||
_dml(update,locals())
|
||||
koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=binfo['state'], new=st_complete, info=binfo)
|
||||
build['id'] = build_id
|
||||
# now to handle the individual rpms
|
||||
for relpath in [srpm] + rpms:
|
||||
fn = "%s/%s" % (uploadpath,relpath)
|
||||
rpminfo = import_rpm(fn,build,brmap.get(relpath))
|
||||
import_rpm_file(fn,build,rpminfo)
|
||||
rpminfo = import_rpm(fn, binfo, brmap.get(relpath))
|
||||
import_rpm_file(fn, binfo, rpminfo)
|
||||
add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn))
|
||||
if logs:
|
||||
for key, files in logs.iteritems():
|
||||
|
|
@ -4050,10 +4127,10 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None)
|
|||
key = None
|
||||
for relpath in files:
|
||||
fn = "%s/%s" % (uploadpath,relpath)
|
||||
import_build_log(fn, build, subdir=key)
|
||||
import_build_log(fn, binfo, subdir=key)
|
||||
koji.plugin.run_callbacks('postImport', type='build', srpm=srpm, rpms=rpms, brmap=brmap,
|
||||
task_id=task_id, build_id=build_id, build=binfo, logs=logs)
|
||||
return build
|
||||
return binfo
|
||||
|
||||
def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False):
|
||||
"""Import a single rpm into the database
|
||||
|
|
@ -4079,14 +4156,11 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False):
|
|||
if buildinfo is None:
|
||||
#figure it out for ourselves
|
||||
if rpminfo['sourcepackage'] == 1:
|
||||
buildinfo = rpminfo.copy()
|
||||
build_id = find_build_id(buildinfo)
|
||||
if build_id:
|
||||
# build already exists
|
||||
buildinfo['id'] = build_id
|
||||
else:
|
||||
buildinfo = get_build(rpminfo, strict=False)
|
||||
if not buildinfo:
|
||||
# create a new build
|
||||
buildinfo['id'] = new_build(rpminfo)
|
||||
build_id = new_build(rpminfo)
|
||||
buildinfo = get_build(build_id, strict=True)
|
||||
else:
|
||||
#figure it out from sourcerpm string
|
||||
buildinfo = get_build(koji.parse_NVRA(rpminfo['sourcerpm']))
|
||||
|
|
@ -4113,29 +4187,27 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False):
|
|||
|
||||
#add rpminfo entry
|
||||
rpminfo['id'] = _singleValue("""SELECT nextval('rpminfo_id_seq')""")
|
||||
rpminfo['build'] = buildinfo
|
||||
rpminfo['build_id'] = buildinfo['id']
|
||||
rpminfo['size'] = os.path.getsize(fn)
|
||||
rpminfo['payloadhash'] = koji.hex_string(hdr[rpm.RPMTAG_SIGMD5])
|
||||
rpminfo['brootid'] = brootid
|
||||
rpminfo['buildroot_id'] = brootid
|
||||
rpminfo['external_repo_id'] = 0
|
||||
|
||||
koji.plugin.run_callbacks('preImport', type='rpm', rpm=rpminfo, build=buildinfo,
|
||||
filepath=fn)
|
||||
|
||||
q = """INSERT INTO rpminfo (id,name,version,release,epoch,
|
||||
build_id,arch,buildtime,buildroot_id,
|
||||
external_repo_id,
|
||||
size,payloadhash)
|
||||
VALUES (%(id)i,%(name)s,%(version)s,%(release)s,%(epoch)s,
|
||||
%(build_id)s,%(arch)s,%(buildtime)s,%(brootid)s,
|
||||
0,
|
||||
%(size)s,%(payloadhash)s)
|
||||
"""
|
||||
_dml(q, rpminfo)
|
||||
data = rpminfo.copy()
|
||||
del data['sourcepackage']
|
||||
del data['sourcerpm']
|
||||
insert = InsertProcessor('rpminfo', data=data)
|
||||
insert.execute()
|
||||
|
||||
koji.plugin.run_callbacks('postImport', type='rpm', rpm=rpminfo, build=buildinfo,
|
||||
filepath=fn)
|
||||
|
||||
#extra fields for return
|
||||
rpminfo['build'] = buildinfo
|
||||
rpminfo['brootid'] = brootid
|
||||
return rpminfo
|
||||
|
||||
def add_external_rpm(rpminfo, external_repo, strict=True):
|
||||
|
|
@ -6897,6 +6969,11 @@ class RootExports(object):
|
|||
def buildReferences(self, build, limit=None):
|
||||
return build_references(get_build(build, strict=True)['id'], limit)
|
||||
|
||||
addVolume = staticmethod(add_volume)
|
||||
removeVolume = staticmethod(remove_volume)
|
||||
listVolumes = staticmethod(list_volumes)
|
||||
changeBuildVolume = staticmethod(change_build_volume)
|
||||
|
||||
def createEmptyBuild(self, name, version, release, epoch, owner=None):
|
||||
context.session.assertPerm('admin')
|
||||
data = { 'name' : name, 'version' : version, 'release' : release,
|
||||
|
|
@ -7364,6 +7441,7 @@ class RootExports(object):
|
|||
return readTaggedArchives(tag, event=event, inherit=inherit, latest=latest, package=package, type=type)
|
||||
|
||||
def listBuilds(self, packageID=None, userID=None, taskID=None, prefix=None, state=None,
|
||||
volumeID=None,
|
||||
createdBefore=None, createdAfter=None,
|
||||
completeBefore=None, completeAfter=None, type=None, typeInfo=None, queryOpts=None):
|
||||
"""List package builds.
|
||||
|
|
@ -7371,7 +7449,8 @@ class RootExports(object):
|
|||
If userID is specified, restrict the results to builds owned by the given user.
|
||||
If taskID is specfied, restrict the results to builds with the given task ID. If taskID is -1,
|
||||
restrict the results to builds with a non-null taskID.
|
||||
One or more of packageID, userID, and taskID may be specified.
|
||||
If volumeID is specified, restrict the results to builds stored on that volume
|
||||
One or more of packageID, userID, volumeID, and taskID may be specified.
|
||||
If prefix is specified, restrict the results to builds whose package name starts with that
|
||||
prefix.
|
||||
If createdBefore and/or createdAfter are specified, restrict the results to builds whose
|
||||
|
|
@ -7400,6 +7479,8 @@ class RootExports(object):
|
|||
- nvr (synthesized for sorting purposes)
|
||||
- owner_id
|
||||
- owner_name
|
||||
- volume_id
|
||||
- volume_name
|
||||
- creation_event_id
|
||||
- creation_time
|
||||
- creation_ts
|
||||
|
|
@ -7421,18 +7502,22 @@ class RootExports(object):
|
|||
('EXTRACT(EPOCH FROM events.time)','creation_ts'),
|
||||
('EXTRACT(EPOCH FROM build.completion_time)','completion_ts'),
|
||||
('package.id', 'package_id'), ('package.name', 'package_name'), ('package.name', 'name'),
|
||||
('volume.id', 'volume_id'), ('volume.name', 'volume_name'),
|
||||
("package.name || '-' || build.version || '-' || build.release", 'nvr'),
|
||||
('users.id', 'owner_id'), ('users.name', 'owner_name')]
|
||||
|
||||
tables = ['build']
|
||||
joins = ['events ON build.create_event = events.id',
|
||||
'package ON build.pkg_id = package.id',
|
||||
'volume ON build.volume_id = volume.id',
|
||||
'users ON build.owner = users.id']
|
||||
clauses = []
|
||||
if packageID != None:
|
||||
clauses.append('package.id = %(packageID)i')
|
||||
if userID != None:
|
||||
clauses.append('users.id = %(userID)i')
|
||||
if volumeID != None:
|
||||
clauses.append('volume.id = %(packageID)i')
|
||||
if taskID != None:
|
||||
if taskID == -1:
|
||||
clauses.append('build.task_id IS NOT NULL')
|
||||
|
|
|
|||
|
|
@ -1396,9 +1396,15 @@ class PathInfo(object):
|
|||
|
||||
topdir = property(topdir, _set_topdir)
|
||||
|
||||
def volumedir(self, volume):
|
||||
if volume == 'DEFAULT' or volume is None:
|
||||
return self.topdir
|
||||
#else
|
||||
return self.topdir + ("/vol/%s" % volume)
|
||||
|
||||
def build(self,build):
|
||||
"""Return the directory where a build belongs"""
|
||||
return self.topdir + ("/packages/%(name)s/%(version)s/%(release)s" % build)
|
||||
return self.volumedir(build.get('volume_name')) + ("/packages/%(name)s/%(version)s/%(release)s" % build)
|
||||
|
||||
def mavenbuild(self, build, maveninfo):
|
||||
"""Return the directory where the Maven build exists in the global store (/mnt/koji/maven2)"""
|
||||
|
|
@ -1406,7 +1412,7 @@ class PathInfo(object):
|
|||
artifact_id = maveninfo['artifact_id']
|
||||
version = maveninfo['version']
|
||||
release = build['release']
|
||||
return self.topdir + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s/%(release)s" % locals())
|
||||
return self.volumedir(build.get('volume_name')) + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s/%(release)s" % locals())
|
||||
|
||||
def winbuild(self, build):
|
||||
"""Return the directory where the Windows build exists"""
|
||||
|
|
|
|||
42
koji/util.py
42
koji/util.py
|
|
@ -19,6 +19,8 @@ import re
|
|||
import time
|
||||
import koji
|
||||
import os
|
||||
import os.path
|
||||
import stat
|
||||
|
||||
try:
|
||||
from hashlib import md5 as md5_constructor
|
||||
|
|
@ -110,6 +112,46 @@ def dslice(dict, keys, strict=True):
|
|||
ret[key] = dict[key]
|
||||
return ret
|
||||
|
||||
|
||||
def rmtree(path):
|
||||
"""Delete a directory tree without crossing fs boundaries"""
|
||||
st = os.lstat(path)
|
||||
if not stat.S_ISDIR(st.st_mode):
|
||||
raise koji.GenericError, "Not a directory: %s" % path
|
||||
dev = st.st_dev
|
||||
dirlist = []
|
||||
for dirpath, dirnames, filenames in os.walk(path):
|
||||
dirlist.append(dirpath)
|
||||
newdirs = []
|
||||
dirsyms = []
|
||||
for fn in dirnames:
|
||||
path = os.path.join(dirpath, fn)
|
||||
st = os.lstat(path)
|
||||
if st.st_dev != dev:
|
||||
# don't cross fs boundary
|
||||
continue
|
||||
if stat.S_ISLNK(st.st_mode):
|
||||
#os.walk includes symlinks to dirs here
|
||||
dirsyms.append(fn)
|
||||
continue
|
||||
newdirs.append(fn)
|
||||
#only walk our filtered dirs
|
||||
dirnames[:] = newdirs
|
||||
for fn in filenames + dirsyms:
|
||||
path = os.path.join(dirpath, fn)
|
||||
st = os.lstat(path)
|
||||
if st.st_dev != dev:
|
||||
#shouldn't happen, but just to be safe...
|
||||
continue
|
||||
os.unlink(path)
|
||||
dirlist.reverse()
|
||||
for dirpath in dirlist:
|
||||
if os.listdir(dirpath):
|
||||
# dir not empty. could happen if a mount was present
|
||||
continue
|
||||
os.rmdir(dirpath)
|
||||
|
||||
|
||||
def eventFromOpts(session, opts):
|
||||
"""Determine event id from standard cli options
|
||||
|
||||
|
|
|
|||
|
|
@ -11,9 +11,7 @@ Alias /koji "/usr/share/koji-web/scripts/"
|
|||
PythonDebug On
|
||||
PythonOption SiteName Koji
|
||||
PythonOption KojiHubURL http://hub.example.com/kojihub
|
||||
PythonOption KojiPackagesURL http://server.example.com/mnt/koji/packages
|
||||
PythonOption KojiMavenURL http://server.example.com/mnt/koji/maven2
|
||||
PythonOption KojiImagesURL http://server.example.com/mnt/koji/images
|
||||
PythonOption KojiFilesURL http://server.example.com/mnt/koji
|
||||
PythonOption WebPrincipal koji/web@EXAMPLE.COM
|
||||
PythonOption WebKeytab /etc/httpd.keytab
|
||||
PythonOption WebCCache /var/tmp/kojiweb.ccache
|
||||
|
|
|
|||
|
|
@ -3,12 +3,10 @@
|
|||
#from kojiweb import util
|
||||
|
||||
#include "includes/header.chtml"
|
||||
#set $nvrpath = '%(name)s/%(version)s/%(release)s' % $build
|
||||
#set $nvrpath = $pathinfo.build($build)
|
||||
#set $archiveurl = ''
|
||||
#if $mavenbuild
|
||||
#set $archiveurl = '%s/%s/%s/%s/%s' % ($mavenBase, $mavenbuild['group_id'].replace('.', '/'),
|
||||
$mavenbuild['artifact_id'], $mavenbuild['version'],
|
||||
$build['release'])
|
||||
#set $archiveurl = $pathinfo.mavenbuild($build,$mavenbuild)
|
||||
#end if
|
||||
|
||||
<h4>Information for build <a href="buildinfo?buildID=$build.id">$koji.buildLabel($build)</a></h4>
|
||||
|
|
@ -108,9 +106,10 @@
|
|||
<tr><th>src</th><th></th></tr>
|
||||
#for $rpm in $rpmsByArch['src']
|
||||
#set $rpmfile = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % $rpm
|
||||
#set $rpmpath = $pathinfo.rpm($rpm)
|
||||
<tr>
|
||||
<td></td>
|
||||
<td>$rpmfile (<a href="rpminfo?rpmID=$rpm.id">info</a>) (<a href="$downloadBase/$nvrpath/$rpm.arch/$rpmfile">download</a>)</td>
|
||||
<td>$rpmfile (<a href="rpminfo?rpmID=$rpm.id">info</a>) (<a href="$nvrpath/$rpmpath">download</a>)</td>
|
||||
</tr>
|
||||
#end for
|
||||
#end if
|
||||
|
|
@ -125,9 +124,9 @@
|
|||
<td>
|
||||
#if $task
|
||||
#if $arch == 'noarch'
|
||||
(<a href="$downloadBase/$nvrpath/data/logs/$noarch_log_dest/">build logs</a>)
|
||||
(<a href="$nvrpath/data/logs/$noarch_log_dest/">build logs</a>)
|
||||
#else
|
||||
(<a href="$downloadBase/$nvrpath/data/logs/$arch/">build logs</a>)
|
||||
(<a href="$nvrpath/data/logs/$arch/">build logs</a>)
|
||||
#end if
|
||||
#end if
|
||||
</td>
|
||||
|
|
@ -135,9 +134,10 @@
|
|||
#for $rpm in $rpmsByArch[$arch] + $debuginfoByArch.get($arch, [])
|
||||
<tr>
|
||||
#set $rpmfile = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % $rpm
|
||||
#set $rpmpath = $pathinfo.rpm($rpm)
|
||||
<td></td>
|
||||
<td>
|
||||
$rpmfile (<a href="rpminfo?rpmID=$rpm.id">info</a>) (<a href="$downloadBase/$nvrpath/$rpm.arch/$rpmfile">download</a>)
|
||||
$rpmfile (<a href="rpminfo?rpmID=$rpm.id">info</a>) (<a href="$nvrpath/$rpmpath">download</a>)
|
||||
</td>
|
||||
</tr>
|
||||
#end for
|
||||
|
|
@ -160,9 +160,9 @@
|
|||
<td>
|
||||
#if $ext == $exts[0]
|
||||
#if $mavenbuild
|
||||
(<a href="$downloadBase/$nvrpath/data/logs/maven2/">build logs</a>)
|
||||
(<a href="$nvrpath/data/logs/maven2/">build logs</a>)
|
||||
#elif $winbuild
|
||||
(<a href="$downloadBase/$nvrpath/data/logs/win/">build logs</a>)
|
||||
(<a href="$nvrpath/data/logs/win/">build logs</a>)
|
||||
#end if
|
||||
#end if
|
||||
</td>
|
||||
|
|
@ -174,7 +174,7 @@
|
|||
#if $mavenbuild
|
||||
$archive.filename (<a href="archiveinfo?archiveID=$archive.id">info</a>) (<a href="$archiveurl/$archive.filename">download</a>)
|
||||
#elif $winbuild
|
||||
$koji.pathinfo.winfile($archive) (<a href="archiveinfo?archiveID=$archive.id">info</a>) (<a href="$downloadBase/$nvrpath/win/$koji.pathinfo.winfile($archive)">download</a>)
|
||||
$pathinfo.winfile($archive) (<a href="archiveinfo?archiveID=$archive.id">info</a>) (<a href="$pathinfo.winbuild($build)/$pathinfo.winfile($archive)">download</a>)
|
||||
#end if
|
||||
</td>
|
||||
</tr>
|
||||
|
|
|
|||
|
|
@ -612,7 +612,7 @@ def imageinfo(req, imageID):
|
|||
"""Do some prep work and generate the imageinfo page for kojiweb."""
|
||||
server = _getServer(req)
|
||||
values = _initValues(req, 'Image Information')
|
||||
imageURL = req.get_options().get('KojiImagesURL', 'http://localhost/images')
|
||||
imageURL = req.get_options().get('KojiFilesURL', 'http://localhost') + '/images'
|
||||
imageID = int(imageID)
|
||||
image = server.getImageInfo(imageID=imageID, strict=True)
|
||||
values['image'] = image
|
||||
|
|
@ -1137,8 +1137,8 @@ def buildinfo(req, buildID):
|
|||
else:
|
||||
values['estCompletion'] = None
|
||||
|
||||
values['downloadBase'] = req.get_options().get('KojiPackagesURL', 'http://localhost/packages')
|
||||
values['mavenBase'] = req.get_options().get('KojiMavenURL', 'http://localhost/maven2')
|
||||
topurl = req.get_options().get('KojiFilesURL', 'http://localhost/')
|
||||
values['pathinfo'] = koji.PathInfo(topdir=topurl)
|
||||
|
||||
return _genHTML(req, 'buildinfo.chtml')
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue