diff --git a/builder/kojid b/builder/kojid index 4346d419..7ddc55bd 100755 --- a/builder/kojid +++ b/builder/kojid @@ -52,7 +52,7 @@ import Cheetah.Template from ConfigParser import ConfigParser from fnmatch import fnmatch from gzip import GzipFile -from optparse import OptionParser +from optparse import OptionParser, SUPPRESS_HELP from StringIO import StringIO #imports for LiveCD and Appliance handler @@ -221,20 +221,8 @@ class BuildRoot(object): repo_id = self.repoid tag_name = self.tag_name - topurl = None - topdir = None - if hasattr(self.options, 'topurl'): - topurl = self.options.topurl - if hasattr(self.options, 'topdir'): - topdir = self.options.topdir - if topurl: - pi = koji.PathInfo(topdir=topurl) - repourl = pi.repo(repo_id, tag_name) + '/maven2' - elif topdir: - pi = koji.PathInfo(topdir=topdir) - repourl = 'file://' + pi.repo(repo_id, tag_name) + '/maven2' - else: - raise koji.BuildError, 'either topurl or topdir must be specified in the config file' + pi = koji.PathInfo(topdir=self.options.topurl) + repourl = pi.repo(repo_id, tag_name) + '/maven2' localrepo = localrepodir[len(self.rootdir()):] settings = """ SetHandler mod_python @@ -15,6 +16,13 @@ Alias /kojihub "/usr/share/koji-hub/XMLRPC" PythonAutoReload Off + + Options Indexes + AllowOverride None + Order allow,deny + Allow from all + + # uncomment this to enable authentication via SSL client certificates # # SSLVerifyClient require diff --git a/hub/kojihub.py b/hub/kojihub.py index 093f465b..d695bc7b 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -40,6 +40,7 @@ import os import random import re import rpm +import shutil import stat import subprocess import sys @@ -1087,6 +1088,7 @@ def readTaggedBuilds(tag,event=None,inherit=False,latest=False,package=None,owne ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), ('build.task_id','task_id'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), + ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), ('package.id', 'package_id'), ('package.name', 'package_name'), ('package.name', 'name'), ("package.name || '-' || build.version || '-' || build.release", 'nvr'), @@ -1115,6 +1117,7 @@ def readTaggedBuilds(tag,event=None,inherit=False,latest=False,package=None,owne JOIN users ON users.id = build.owner JOIN events ON events.id = build.create_event JOIN package ON package.id = build.pkg_id + JOIN volume ON volume.id = build.volume_id WHERE %s AND tag_id=%%(tagid)s AND build.state=%%(st_complete)i """ % (', '.join([pair[0] for pair in fields]), type_join, eventCondition(event, 'tag_listing')) @@ -2080,6 +2083,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None): packages = {} for repoarch in repo_arches: packages.setdefault(repoarch, []) + relpathinfo = koji.PathInfo(topdir='') for rpminfo in rpms: if not with_debuginfo and koji.is_debuginfo(rpminfo['name']): continue @@ -2094,7 +2098,8 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None): # Do not create a repo for arches not in the arch list for this tag continue build = builds[rpminfo['build_id']] - rpminfo['path'] = "%s/%s" % (koji.pathinfo.build(build), koji.pathinfo.rpm(rpminfo)) + rpminfo['relpath'] = "%s/%s" % (relpathinfo.build(build), relpathinfo.rpm(rpminfo)) + rpminfo['relpath'] = rpminfo['relpath'].lstrip('/') packages.setdefault(repoarch,[]).append(rpminfo) #generate comps and groups.spec groupsdir = "%s/groups" % (repodir) @@ -2107,7 +2112,7 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None): if context.opts.get('EnableMaven') and tinfo['maven_support']: maven_builds = maven_tag_packages(tinfo, event_id) - #link packages + #generate pkglist files for arch in packages.iterkeys(): if arch in ['src','noarch']: continue @@ -2117,16 +2122,16 @@ def repo_init(tag, with_src=False, with_debuginfo=False, event=None): pkglist = file(os.path.join(repodir, arch, 'pkglist'), 'w') logger.info("Creating package list for %s" % arch) for rpminfo in packages[arch]: - pkglist.write(rpminfo['path'].split(os.path.join(koji.pathinfo.topdir, 'packages/'))[1] + '\n') + pkglist.write(rpminfo['relpath'] + '\n') #noarch packages for rpminfo in packages.get('noarch',[]): - pkglist.write(rpminfo['path'].split(os.path.join(koji.pathinfo.topdir, 'packages/'))[1] + '\n') + pkglist.write(rpminfo['relpath'] + '\n') # srpms if with_src: srpmdir = "%s/%s" % (repodir,'src') koji.ensuredir(srpmdir) for rpminfo in packages.get('src',[]): - pkglist.write(rpminfo['path'].split(os.path.join(koji.pathinfo.topdir, 'packages/'))[1] + '\n') + pkglist.write(rpminfo['relpath'] + '\n') pkglist.close() #write list of blocked packages blocklist = file(os.path.join(repodir, arch, 'blocklist'), 'w') @@ -3066,6 +3071,8 @@ def get_build(buildInfo, strict=False): task_id: ID of the task that kicked off the build owner_id: ID of the user who kicked off the build owner_name: name of the user who kicked off the build + volume_id: ID of the storage volume + volume_name: name of the storage volume creation_event_id: id of the create_event creation_time: time the build was created (text) creation_ts: time the build was created (epoch) @@ -3086,6 +3093,7 @@ def get_build(buildInfo, strict=False): ('build.epoch', 'epoch'), ('build.state', 'state'), ('build.completion_time', 'completion_time'), ('build.task_id', 'task_id'), ('events.id', 'creation_event_id'), ('events.time', 'creation_time'), ('package.id', 'package_id'), ('package.name', 'package_name'), ('package.name', 'name'), + ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), ("package.name || '-' || build.version || '-' || build.release", 'nvr'), ('EXTRACT(EPOCH FROM events.time)','creation_ts'), ('EXTRACT(EPOCH FROM build.completion_time)','completion_ts'), @@ -3094,6 +3102,7 @@ def get_build(buildInfo, strict=False): FROM build JOIN events ON build.create_event = events.id JOIN package on build.pkg_id = package.id + JOIN volume on build.volume_id = volume.id JOIN users on build.owner = users.id WHERE build.id = %%(buildID)i""" % ', '.join([pair[0] for pair in fields]) @@ -3891,6 +3900,77 @@ def new_package(name,strict=True): c.execute(q,locals()) return pkg_id + +def add_volume(name, strict=True): + """Add a new storage volume in the database""" + context.session.assertPerm('admin') + if strict: + volinfo = lookup_name('volume', name, strict=False) + if volinfo: + raise koji.GenericError, 'volume %s already exists' % name + volinfo = lookup_name('volume', name, strict=False, create=True) + +def remove_volume(volume): + """Remove unused storage volume from the database""" + context.session.assertPerm('admin') + volinfo = lookup_name('volume', volume, strict=True) + query = QueryProcessor(tables=['build'], clauses=['volume_id=%(id)i'], + values=volinfo, columns=['id'], opts={'limit':1}) + if query.execute(): + raise koji.GenericError, 'volume %(name)s has build references' % volinfo + delete = """DELETE FROM volume WHERE id=%(id)i""" + _dml(delete, volinfo) + +def list_volumes(): + """List storage volumes""" + return QueryProcessor(tables=['volume'], columns=['id', 'name']).execute() + +def change_build_volume(build, volume, strict=True): + """Move a build to a different storage volume""" + context.session.assertPerm('admin') + volinfo = lookup_name('volume', volume, strict=True) + binfo = get_build(build, strict=True) + if binfo['volume_id'] == volinfo['id']: + if strict: + raise koji.GenericError, "Build %(nvr)s already on volume %(volume_name)s" % binfo + else: + #nothing to do + return + state = koji.BUILD_STATES[binfo['state']] + if state not in ['COMPLETE', 'DELETED']: + raise koji.GenericError, "Build %s is %s" % (binfo['nvr'], state) + + # First copy the build dir(s) + dir_moves = [] + old_binfo = binfo.copy() + binfo['volume_id'] = volinfo['id'] + binfo['volume_name'] = volinfo['name'] + olddir = koji.pathinfo.build(old_binfo) + if os.path.exists(olddir): + newdir = koji.pathinfo.build(binfo) + dir_moves.append([olddir, newdir]) + maven_info = get_maven_build(binfo['id']) + if maven_info: + olddir = koji.pathinfo.mavenbuild(old_binfo, maven_info) + if os.path.exists(olddir): + newdir = koji.pathinfo.mavenbuild(binfo, maven_info) + dir_moves.append([olddir, newdir]) + for olddir, newdir in dir_moves: + shutil.copytree(olddir, newdir, symlinks=True) + + # Second, update the db + koji.plugin.run_callbacks('preBuildStateChange', attribute='volume_id', old=old_binfo['volume_id'], new=volinfo['id'], info=binfo) + update = UpdateProcessor('build', clauses=['id=%(id)i'], values=binfo) + update.set(volume_id=volinfo['id']) + update.execute() + + # Third, delete the old content + for olddir, newdir in dir_moves: + koji.util.rmtree(olddir) + + koji.plugin.run_callbacks('postBuildStateChange', attribute='volume_id', old=old_binfo['volume_id'], new=volinfo['id'], info=binfo) + + def new_build(data): """insert a new build entry""" data = data.copy() @@ -3908,6 +3988,7 @@ def new_build(data): data.setdefault('completion_time', 'NOW') data.setdefault('owner',context.session.user_id) data.setdefault('task_id',None) + data.setdefault('volume_id', 0) #check for existing build # TODO - table lock? q="""SELECT id,state,task_id FROM build @@ -3939,14 +4020,11 @@ def new_build(data): else: koji.plugin.run_callbacks('preBuildStateChange', attribute='state', old=None, new=data['state'], info=data) #insert the new data + data = dslice(data, ['pkg_id', 'version', 'release', 'epoch', 'state', 'volume_id', + 'task_id', 'owner', 'completion_time']) data['id'] = _singleValue("SELECT nextval('build_id_seq')") - q=""" - INSERT INTO build (id,pkg_id,version,release,epoch,state, - task_id,owner,completion_time) - VALUES (%(id)i,%(pkg_id)i,%(version)s,%(release)s,%(epoch)s, - %(state)s,%(task_id)s,%(owner)s,%(completion_time)s) - """ - _dml(q, data) + insert = InsertProcessor('build', data=data) + insert.execute() koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=None, new=data['state'], info=data) #return build_id return data['id'] @@ -4037,12 +4115,11 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) WHERE id=%(build_id)i""" _dml(update,locals()) koji.plugin.run_callbacks('postBuildStateChange', attribute='state', old=binfo['state'], new=st_complete, info=binfo) - build['id'] = build_id # now to handle the individual rpms for relpath in [srpm] + rpms: fn = "%s/%s" % (uploadpath,relpath) - rpminfo = import_rpm(fn,build,brmap.get(relpath)) - import_rpm_file(fn,build,rpminfo) + rpminfo = import_rpm(fn, binfo, brmap.get(relpath)) + import_rpm_file(fn, binfo, rpminfo) add_rpm_sig(rpminfo['id'], koji.rip_rpm_sighdr(fn)) if logs: for key, files in logs.iteritems(): @@ -4050,10 +4127,10 @@ def import_build(srpm, rpms, brmap=None, task_id=None, build_id=None, logs=None) key = None for relpath in files: fn = "%s/%s" % (uploadpath,relpath) - import_build_log(fn, build, subdir=key) + import_build_log(fn, binfo, subdir=key) koji.plugin.run_callbacks('postImport', type='build', srpm=srpm, rpms=rpms, brmap=brmap, task_id=task_id, build_id=build_id, build=binfo, logs=logs) - return build + return binfo def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False): """Import a single rpm into the database @@ -4079,14 +4156,11 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False): if buildinfo is None: #figure it out for ourselves if rpminfo['sourcepackage'] == 1: - buildinfo = rpminfo.copy() - build_id = find_build_id(buildinfo) - if build_id: - # build already exists - buildinfo['id'] = build_id - else: + buildinfo = get_build(rpminfo, strict=False) + if not buildinfo: # create a new build - buildinfo['id'] = new_build(rpminfo) + build_id = new_build(rpminfo) + buildinfo = get_build(build_id, strict=True) else: #figure it out from sourcerpm string buildinfo = get_build(koji.parse_NVRA(rpminfo['sourcerpm'])) @@ -4113,29 +4187,27 @@ def import_rpm(fn,buildinfo=None,brootid=None,wrapper=False): #add rpminfo entry rpminfo['id'] = _singleValue("""SELECT nextval('rpminfo_id_seq')""") - rpminfo['build'] = buildinfo rpminfo['build_id'] = buildinfo['id'] rpminfo['size'] = os.path.getsize(fn) rpminfo['payloadhash'] = koji.hex_string(hdr[rpm.RPMTAG_SIGMD5]) - rpminfo['brootid'] = brootid + rpminfo['buildroot_id'] = brootid + rpminfo['external_repo_id'] = 0 koji.plugin.run_callbacks('preImport', type='rpm', rpm=rpminfo, build=buildinfo, filepath=fn) - q = """INSERT INTO rpminfo (id,name,version,release,epoch, - build_id,arch,buildtime,buildroot_id, - external_repo_id, - size,payloadhash) - VALUES (%(id)i,%(name)s,%(version)s,%(release)s,%(epoch)s, - %(build_id)s,%(arch)s,%(buildtime)s,%(brootid)s, - 0, - %(size)s,%(payloadhash)s) - """ - _dml(q, rpminfo) + data = rpminfo.copy() + del data['sourcepackage'] + del data['sourcerpm'] + insert = InsertProcessor('rpminfo', data=data) + insert.execute() koji.plugin.run_callbacks('postImport', type='rpm', rpm=rpminfo, build=buildinfo, filepath=fn) + #extra fields for return + rpminfo['build'] = buildinfo + rpminfo['brootid'] = brootid return rpminfo def add_external_rpm(rpminfo, external_repo, strict=True): @@ -6897,6 +6969,11 @@ class RootExports(object): def buildReferences(self, build, limit=None): return build_references(get_build(build, strict=True)['id'], limit) + addVolume = staticmethod(add_volume) + removeVolume = staticmethod(remove_volume) + listVolumes = staticmethod(list_volumes) + changeBuildVolume = staticmethod(change_build_volume) + def createEmptyBuild(self, name, version, release, epoch, owner=None): context.session.assertPerm('admin') data = { 'name' : name, 'version' : version, 'release' : release, @@ -7364,6 +7441,7 @@ class RootExports(object): return readTaggedArchives(tag, event=event, inherit=inherit, latest=latest, package=package, type=type) def listBuilds(self, packageID=None, userID=None, taskID=None, prefix=None, state=None, + volumeID=None, createdBefore=None, createdAfter=None, completeBefore=None, completeAfter=None, type=None, typeInfo=None, queryOpts=None): """List package builds. @@ -7371,7 +7449,8 @@ class RootExports(object): If userID is specified, restrict the results to builds owned by the given user. If taskID is specfied, restrict the results to builds with the given task ID. If taskID is -1, restrict the results to builds with a non-null taskID. - One or more of packageID, userID, and taskID may be specified. + If volumeID is specified, restrict the results to builds stored on that volume + One or more of packageID, userID, volumeID, and taskID may be specified. If prefix is specified, restrict the results to builds whose package name starts with that prefix. If createdBefore and/or createdAfter are specified, restrict the results to builds whose @@ -7400,6 +7479,8 @@ class RootExports(object): - nvr (synthesized for sorting purposes) - owner_id - owner_name + - volume_id + - volume_name - creation_event_id - creation_time - creation_ts @@ -7421,18 +7502,22 @@ class RootExports(object): ('EXTRACT(EPOCH FROM events.time)','creation_ts'), ('EXTRACT(EPOCH FROM build.completion_time)','completion_ts'), ('package.id', 'package_id'), ('package.name', 'package_name'), ('package.name', 'name'), + ('volume.id', 'volume_id'), ('volume.name', 'volume_name'), ("package.name || '-' || build.version || '-' || build.release", 'nvr'), ('users.id', 'owner_id'), ('users.name', 'owner_name')] tables = ['build'] joins = ['events ON build.create_event = events.id', 'package ON build.pkg_id = package.id', + 'volume ON build.volume_id = volume.id', 'users ON build.owner = users.id'] clauses = [] if packageID != None: clauses.append('package.id = %(packageID)i') if userID != None: clauses.append('users.id = %(userID)i') + if volumeID != None: + clauses.append('volume.id = %(packageID)i') if taskID != None: if taskID == -1: clauses.append('build.task_id IS NOT NULL') diff --git a/koji/__init__.py b/koji/__init__.py index 97a29749..4c137b70 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1396,9 +1396,15 @@ class PathInfo(object): topdir = property(topdir, _set_topdir) + def volumedir(self, volume): + if volume == 'DEFAULT' or volume is None: + return self.topdir + #else + return self.topdir + ("/vol/%s" % volume) + def build(self,build): """Return the directory where a build belongs""" - return self.topdir + ("/packages/%(name)s/%(version)s/%(release)s" % build) + return self.volumedir(build.get('volume_name')) + ("/packages/%(name)s/%(version)s/%(release)s" % build) def mavenbuild(self, build, maveninfo): """Return the directory where the Maven build exists in the global store (/mnt/koji/maven2)""" @@ -1406,7 +1412,7 @@ class PathInfo(object): artifact_id = maveninfo['artifact_id'] version = maveninfo['version'] release = build['release'] - return self.topdir + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s/%(release)s" % locals()) + return self.volumedir(build.get('volume_name')) + ("/maven2/%(group_path)s/%(artifact_id)s/%(version)s/%(release)s" % locals()) def winbuild(self, build): """Return the directory where the Windows build exists""" diff --git a/koji/util.py b/koji/util.py index 22118e03..7c4953e9 100644 --- a/koji/util.py +++ b/koji/util.py @@ -19,6 +19,8 @@ import re import time import koji import os +import os.path +import stat try: from hashlib import md5 as md5_constructor @@ -110,6 +112,46 @@ def dslice(dict, keys, strict=True): ret[key] = dict[key] return ret + +def rmtree(path): + """Delete a directory tree without crossing fs boundaries""" + st = os.lstat(path) + if not stat.S_ISDIR(st.st_mode): + raise koji.GenericError, "Not a directory: %s" % path + dev = st.st_dev + dirlist = [] + for dirpath, dirnames, filenames in os.walk(path): + dirlist.append(dirpath) + newdirs = [] + dirsyms = [] + for fn in dirnames: + path = os.path.join(dirpath, fn) + st = os.lstat(path) + if st.st_dev != dev: + # don't cross fs boundary + continue + if stat.S_ISLNK(st.st_mode): + #os.walk includes symlinks to dirs here + dirsyms.append(fn) + continue + newdirs.append(fn) + #only walk our filtered dirs + dirnames[:] = newdirs + for fn in filenames + dirsyms: + path = os.path.join(dirpath, fn) + st = os.lstat(path) + if st.st_dev != dev: + #shouldn't happen, but just to be safe... + continue + os.unlink(path) + dirlist.reverse() + for dirpath in dirlist: + if os.listdir(dirpath): + # dir not empty. could happen if a mount was present + continue + os.rmdir(dirpath) + + def eventFromOpts(session, opts): """Determine event id from standard cli options diff --git a/www/conf/kojiweb.conf b/www/conf/kojiweb.conf index 503cdc94..96c8dd78 100644 --- a/www/conf/kojiweb.conf +++ b/www/conf/kojiweb.conf @@ -11,9 +11,7 @@ Alias /koji "/usr/share/koji-web/scripts/" PythonDebug On PythonOption SiteName Koji PythonOption KojiHubURL http://hub.example.com/kojihub - PythonOption KojiPackagesURL http://server.example.com/mnt/koji/packages - PythonOption KojiMavenURL http://server.example.com/mnt/koji/maven2 - PythonOption KojiImagesURL http://server.example.com/mnt/koji/images + PythonOption KojiFilesURL http://server.example.com/mnt/koji PythonOption WebPrincipal koji/web@EXAMPLE.COM PythonOption WebKeytab /etc/httpd.keytab PythonOption WebCCache /var/tmp/kojiweb.ccache diff --git a/www/kojiweb/buildinfo.chtml b/www/kojiweb/buildinfo.chtml index 5ec393ae..10a3cb8e 100644 --- a/www/kojiweb/buildinfo.chtml +++ b/www/kojiweb/buildinfo.chtml @@ -3,12 +3,10 @@ #from kojiweb import util #include "includes/header.chtml" -#set $nvrpath = '%(name)s/%(version)s/%(release)s' % $build +#set $nvrpath = $pathinfo.build($build) #set $archiveurl = '' #if $mavenbuild -#set $archiveurl = '%s/%s/%s/%s/%s' % ($mavenBase, $mavenbuild['group_id'].replace('.', '/'), - $mavenbuild['artifact_id'], $mavenbuild['version'], - $build['release']) +#set $archiveurl = $pathinfo.mavenbuild($build,$mavenbuild) #end if

Information for build $koji.buildLabel($build)

@@ -108,9 +106,10 @@ src #for $rpm in $rpmsByArch['src'] #set $rpmfile = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % $rpm + #set $rpmpath = $pathinfo.rpm($rpm) - $rpmfile (info) (download) + $rpmfile (info) (download) #end for #end if @@ -125,9 +124,9 @@ #if $task #if $arch == 'noarch' - (build logs) + (build logs) #else - (build logs) + (build logs) #end if #end if @@ -135,9 +134,10 @@ #for $rpm in $rpmsByArch[$arch] + $debuginfoByArch.get($arch, []) #set $rpmfile = '%(name)s-%(version)s-%(release)s.%(arch)s.rpm' % $rpm + #set $rpmpath = $pathinfo.rpm($rpm) - $rpmfile (info) (download) + $rpmfile (info) (download) #end for @@ -160,9 +160,9 @@ #if $ext == $exts[0] #if $mavenbuild - (build logs) + (build logs) #elif $winbuild - (build logs) + (build logs) #end if #end if @@ -174,7 +174,7 @@ #if $mavenbuild $archive.filename (info) (download) #elif $winbuild - $koji.pathinfo.winfile($archive) (info) (download) + $pathinfo.winfile($archive) (info) (download) #end if diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 9261adb6..342ce3a2 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -612,7 +612,7 @@ def imageinfo(req, imageID): """Do some prep work and generate the imageinfo page for kojiweb.""" server = _getServer(req) values = _initValues(req, 'Image Information') - imageURL = req.get_options().get('KojiImagesURL', 'http://localhost/images') + imageURL = req.get_options().get('KojiFilesURL', 'http://localhost') + '/images' imageID = int(imageID) image = server.getImageInfo(imageID=imageID, strict=True) values['image'] = image @@ -1137,8 +1137,8 @@ def buildinfo(req, buildID): else: values['estCompletion'] = None - values['downloadBase'] = req.get_options().get('KojiPackagesURL', 'http://localhost/packages') - values['mavenBase'] = req.get_options().get('KojiMavenURL', 'http://localhost/maven2') + topurl = req.get_options().get('KojiFilesURL', 'http://localhost/') + values['pathinfo'] = koji.PathInfo(topdir=topurl) return _genHTML(req, 'buildinfo.chtml')