particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into shadow

Browse source 
Conflicts:

	builder/kojid
	koji.spec
	util/Makefile
This commit is contained in:
Mike McLean 2008-05-22 16:04:50 -04:00
commit caa1c35bec
5 changed files with 213 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

109
builder/kojid
View file

@ -636,7 +636,8 @@ class TaskManager(object):
else:
self.logger.warn("%s: %s" % (desc, e))
continue
age = min(age, time.time() - st.st_mtime)
else:
age = min(age, time.time() - st.st_mtime)
#note: https://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=192153)
#If rpmlib is installing in this chroot, removing it entirely
#can lead to a world of hurt.
@ -1441,16 +1442,22 @@ class ChainBuildTask(BaseTaskHandler):
for build_level in srcs:
subtasks = []
build_tasks = []
nvrs = []
for src in build_level:
if SCM.is_scm_url(src):
task_id = session.host.subtask(method='build',
arglist=[src, target, opts],
label=src,
parent=self.id)
build_tasks.append(task_id)
subtasks.append(task_id)
else:
task_id = session.host.subtask(method='waitrepo',
arglist=[target, None, src],
parent=self.id)
nvrs.append(src)
if nvrs:
task_id = session.host.subtask(method='waitrepo',
arglist=[target, None, nvrs],
label=','.join(nvrs),
parent=self.id)
subtasks.append(task_id)
if not subtasks:
continue
@ -1464,7 +1471,8 @@ class ChainBuildTask(BaseTaskHandler):
nvrs.append(builds[0]['nvr'])
if nvrs:
task_id = session.host.subtask(method='waitrepo',
arglist=[target_info['build_tag'], None, nvrs],
arglist=[target, None, nvrs],
label=','.join(nvrs),
parent=self.id)
self.wait(task_id, all=True, failany=True)
@ -2329,66 +2337,47 @@ class WaitrepoTask(BaseTaskHandler):
_taskWeight = 0.2
PAUSE = 60
# time in seconds before we fail this task
TIMEOUT = 7200
# time in minutes before we fail this task
TIMEOUT = 120
def checkForNVR(self, tag, nvrs, repo_creation_event):
if nvrs is None:
#check not requested
return True
if not isinstance(nvrs, list):
nvrs = [nvrs]
repo_nvrs = dict([(b['nvr'], 1) for b in session.listTagged(tag, event=repo_creation_event, inherit=True)])
for nvr in nvrs:
if not repo_nvrs.has_key(nvr):
return False
return True
def checkNewerThan(create_ts, newer_than):
if newer_than is None:
#check not requested
return True
return (create_ts > newer_than)
def handler(self, tag, newer_than=None, nvr=None):
"""Wait for a repo for the tag, subject to given conditions
newer_than: create_event timestamp should be newer than this
nvr: repo should contain this nvr (which may not exist at first)
If both of these are unspecified/None, then the call will wait
for the first ready repo.
Returns the repo info (from getRepo) of the chosen repo
"""
def handler(self, build_target_info, newer_then=None, nvrs=None):
start = time.time()
repo = None
build_target = session.getBuildTarget(build_target_info)
if not build_target:
raise koji.GenericError, "invalid build target: %s" % build_target_info
if not nvrs:
nvrs = []
builds = [koji.parse_NVR(nvr) for nvr in nvrs]
if not newer_then and not builds:
newer_then = time.time()
last_repo = None
if isinstance(newer_than, basestring) and newer_than.lower() == "now":
newer_than = start
if not isinstance(newer_than, (int, long, float)):
raise koji.GenericError("Invalid value for newer_than")
taginfo = session.getTag(tag)
#sanity check
targets = session.getBuildTargets(buildTagID=taginfo['id'])
if not targets:
raise koji.GenericError("No build target for tag: %s" % taginfo['name'])
repo = session.getRepo(build_target['build_tag'])
while True:
repo = session.getRepo(taginfo['id'])
#note: getRepo will only return a repo in the "READY" state
if repo and repo != last_repo:
if self.checkNewerThan(repo['create_ts'], newer_than) \
and self.checkForNVR(taginfo['id'], nvr, repo['create_event']):
#note that the self.check* calls return True if the check is not
#requested (i.e. value is None). Also note that the more expensive
#check is listed second.
break
if self.TIMEOUT and ((time.time() - start) > self.TIMEOUT):
raise koji.GenericError("Timed out waiting for repo after %d seconds" % self.TIMEOUT)
last_repo = repo
if builds and repo and repo != last_repo:
if koji.util.checkForBuilds(session, build_target['build_tag'], builds, repo['create_event']):
return "Successfully waited %s for %s to appear in the %s repo" % \
(koji.util.duration(start), koji.util.printList(nvrs), build_target['build_tag_name'])
elif newer_then:
if repo['create_ts'] > newer_then:
return "Successfully waited %s for a new %s repo" % \
(koji.util.duration(start), build_target['build_tag_name'])
if (time.time() - start) > (self.TIMEOUT * 60.0):
if builds:
raise koji.GenericError, "Unsuccessfully waited %s for %s to appear in the %s repo" % \
(koji.util.duration(start), koji.util.printList(nvrs), build_target['build_tag_name'])
else:
raise koji.GenericError, "Unsuccessfully waited %s for a new %s repo" % \
(koji.util.duration(start), build_target['build_tag_name'])
time.sleep(self.PAUSE)
self.logger.debug("Successfully waited %s seconds for a '%s' repo (%s)" \
% ((time.time() - start), taginfo['name'], repo['id']))
return repo
last_repo = repo
repo = session.getRepo(build_target['build_tag'])
class SCM(object):
"SCM abstraction class"

151
cli/koji
View file

@ -688,7 +688,7 @@ def handle_build(options, session, args):
callback = None
else:
callback = _progress_callback
session.uploadWrapper(source, serverdir, callback=callback, blocksize=65536)
session.uploadWrapper(source, serverdir, callback=callback)
print
source = "%s/%s" % (serverdir, os.path.basename(source))
task_id = session.build(source, target, opts, priority=priority)
@ -1028,7 +1028,7 @@ def handle_import(options, session, args):
else:
print _("uploading %s...") % path,
sys.stdout.flush()
session.uploadWrapper(path, serverdir, blocksize=65536)
session.uploadWrapper(path, serverdir)
print _("done")
sys.stdout.flush()
print _("importing %s...") % path,
@ -1211,6 +1211,7 @@ def handle_prune_signed_copies(options, session, args):
parser.add_option("-v", "--verbose", action="store_true", help=_("Be more verbose"))
parser.add_option("--days", type="int", default=5, help=_("Timeout before clearing"))
parser.add_option("-p", "--package", "--pkg", help=_("Limit to a single package"))
parser.add_option("-b", "--build", help=_("Limit to a single build"))
parser.add_option("--trashcan-tag", default="trashcan", help=_("Specify trashcan tag"))
parser.add_option("--debug", action="store_true", help=_("Show debugging output"))
(options, args) = parser.parse_args(args)
@ -1229,16 +1230,24 @@ def handle_prune_signed_copies(options, session, args):
cutoff_ts = time.time() - options.days * 24 * 3600
if options.debug:
print "Cutoff date: %s" % time.asctime(time.localtime(cutoff_ts))
if options.verbose:
print "Getting builds..."
qopts = {'state' : koji.BUILD_STATES['COMPLETE']}
if options.package:
pkginfo = session.getPackage(options.package)
qopts['packageID'] = pkginfo['id']
builds = [(b['nvr'], b) for b in session.listBuilds(**qopts)]
if options.verbose:
print "...got %i builds" % len(builds)
builds.sort()
if not options.build:
if options.verbose:
print "Getting builds..."
qopts = {'state' : koji.BUILD_STATES['COMPLETE']}
if options.package:
pkginfo = session.getPackage(options.package)
qopts['packageID'] = pkginfo['id']
builds = [(b['nvr'], b) for b in session.listBuilds(**qopts)]
if options.verbose:
print "...got %i builds" % len(builds)
builds.sort()
else:
#single build
binfo = session.getBuild(options.build)
if not binfo:
parser.error('No such build: %s' % options.build)
assert False
builds = [("%(name)s-%(version)s-%(release)s" % binfo, binfo)]
total_files = 0
total_space = 0
def _histline(event_id, x):
@ -1287,16 +1296,17 @@ def handle_prune_signed_copies(options, session, args):
for x in hist:
#note that for revoked entries, we're effectively splitting them into
#two parts: creation and revocation.
timeline.append((x['create_event'], 0, x))
timeline.append((x['create_event'], 1, x))
#at the same event, revokes happen first
if x['revoke_event'] is not None:
timeline.append((x['revoke_event'], 1, x))
timeline.append((x['revoke_event'], 0, x))
timeline.sort()
#find most recent creation entry for our build and crop there
latest_ts = None
for i in xrange(len(timeline)-1, -1, -1):
#searching in reverse cronological order
event_id, revoked, entry = timeline[i]
if entry['build_id'] == binfo['id'] and not revoked:
event_id, is_create, entry = timeline[i]
if entry['build_id'] == binfo['id'] and is_create:
latest_ts = event_id
break
if not latest_ts:
@ -1314,7 +1324,7 @@ def handle_prune_signed_copies(options, session, args):
replaced_ts = None
revoke_ts = None
others = {}
for event_id, revoked, entry in timeline:
for event_id, is_create, entry in timeline:
#So two things can knock this build from the title of latest:
# - it could be untagged (entry revoked)
# - another build could become latest (replaced)
@ -1323,7 +1333,7 @@ def handle_prune_signed_copies(options, session, args):
if options.debug:
print _histline(event_id, entry)
if entry['build_id'] == binfo['id']:
if not revoked:
if is_create:
#shouldn't happen
raise koji.GenericError, "Duplicate creation event found for %s in %s" \
% (nvr, tag_name)
@ -1332,7 +1342,7 @@ def handle_prune_signed_copies(options, session, args):
revoke_ts = entry['revoke_ts']
break
else:
if not revoked:
if is_create:
#this build has become latest
replaced_ts = entry['create_ts']
if entry['active']:
@ -2161,7 +2171,11 @@ def anon_handle_list_pkgs(options, session, args):
# no limiting clauses were specified
allpkgs = True
opts['inherited'] = not options.noinherit
opts['with_dups'] = options.show_dups
#hiding dups only makes sense if we're querying a tag
if options.tag:
opts['with_dups'] = options.show_dups
else:
opts['with_dups'] = True
data = session.listPackages(**opts)
if not data:
print "(no matching packages)"
@ -3772,12 +3786,14 @@ def handle_unblock_pkg(options, session, args):
def anon_handle_download_build(options, session, args):
"Download a built package"
usage = _("usage: %prog download-build [options] <n-v-r | build_id>")
usage = _("usage: %prog download-build [options] <n-v-r | build_id | package>")
usage += _("\n(Specify the --help global option for a list of other help options)")
parser = OptionParser(usage=usage)
parser.add_option("--arch", dest="arches", metavar="ARCH", action="append", default=[],
help=_("Only download packages for this arch (may be used multiple times)"))
parser.add_option("--latestfrom", dest="latestfrom", help=_("Download the latest build from this tag"))
parser.add_option("--debuginfo", action="store_true", help=_("Also download -debuginfo rpms"))
parser.add_option("--key", help=_("Download rpms signed with the given key"))
parser.add_option("-q", "--quiet", action="store_true", help=_("Do not display progress meter"))
(suboptions, args) = parser.parse_args(args)
if len(args) < 1:
@ -3791,8 +3807,25 @@ def anon_handle_download_build(options, session, args):
build = args[0]
if build.isdigit():
if suboptions.latestfrom:
print "--latestfrom not compatible with build IDs, specify a package name."
return 1
build = int(build)
info = session.getBuild(build)
if suboptions.latestfrom:
# We want the latest build, not a specific build
try:
builds = session.listTagged(suboptions.latestfrom, latest=True, package=build)
except koji.GenericError, data:
print "Error finding latest build: %s" % data
return 1
if not builds:
print "%s has no builds of %s" % (suboptions.latestfrom, build)
return 1
info = builds[0]
else:
info = session.getBuild(build)
if info is None:
print "No such build: %s" % build
return 1
@ -3816,13 +3849,17 @@ def anon_handle_download_build(options, session, args):
for rpm in rpms:
if rpm['name'].endswith('-debuginfo') and not suboptions.debuginfo:
continue
fname = "%s-%s-%s.%s.rpm" % (rpm['name'], rpm['version'], rpm['release'], rpm['arch'])
url = "%s/%s/%s/%s/%s/%s" % (options.pkgurl, info['package_name'], rpm['version'], rpm['release'], rpm['arch'], fname)
if suboptions.key:
fname = koji.pathinfo.signed(rpm, suboptions.key)
else:
fname = koji.pathinfo.rpm(rpm)
url = "%s/%s/%s/%s/%s" % (options.pkgurl, info['package_name'], info['version'], info['release'], fname)
file = grabber.urlopen(url, progress_obj = pg, text = "%s.%s" % (rpm['name'], rpm['arch']))
out = os.open(fname, os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0666)
out = os.open(os.path.basename(fname), os.O_WRONLY|os.O_CREAT|os.O_TRUNC, 0666)
try:
while 1:
buf = file.read(4096)
@ -3833,6 +3870,68 @@ def anon_handle_download_build(options, session, args):
os.close(out)
file.close()
def anon_handle_wait_repo(options, session, args):
"Wait for a repo to be regenerated"
usage = _("usage: %prog wait-repo [options] <tag>")
usage += _("\n(Specify the --help global option for a list of other help options)")
parser = OptionParser(usage=usage)
parser.add_option("--build", metavar="NVR", dest="builds", action="append", default=[],
help=_("Check that the given build (or a later build of the same package) is in the newly-generated repo (may be used multiple times)"))
parser.add_option("--target", action="store_true", help=_("Interpret the argument as a build target name"))
parser.add_option("--timeout", type="int", help=_("Amount of time to wait (in minutes) before giving up (default: 120)"), default=120)
parser.add_option("--quiet", action="store_true", help=_("Suppress output, success or failure will be indicated by the return value only"))
(suboptions, args) = parser.parse_args(args)
start = time.time()
builds = [koji.parse_NVR(build) for build in suboptions.builds]
if len(args) < 1:
parser.error(_("Please specify a tag name"))
elif len(args) > 1:
parser.error(_("Only one tag may be specified"))
tag = args[0]
if suboptions.target:
target_info = session.getBuildTarget(tag)
if not target_info:
parser.error("Invalid build target: %s" % tag)
tag = target_info['build_tag_name']
tag_id = target_info['build_tag']
else:
tag_info = session.getTag(tag)
if not tag_info:
parser.error("Invalid tag: %s" % tag)
tag_id = tag_info['id']
last_repo = None
repo = session.getRepo(tag_id)
while True:
if builds and repo and repo != last_repo:
if koji.util.checkForBuilds(session, tag_id, builds, repo['create_event']):
if not suboptions.quiet:
print "Successfully waited %s for %s to appear in the %s repo" % (koji.util.duration(start), koji.util.printList(suboptions.builds), tag)
return
time.sleep(60)
last_repo = repo
repo = session.getRepo(tag_id)
if not builds:
if repo != last_repo:
if not suboptions.quiet:
print "Successfully waited %s for a new %s repo" % (koji.util.duration(start), tag)
return
if (time.time() - start) > (suboptions.timeout * 60.0):
if not suboptions.quiet:
if builds:
print "Unsuccessfully waited %s for %s to appear in the %s repo" % (koji.util.duration(start), koji.util.printList(suboptions.builds), tag)
else:
print "Unsuccessfully waited %s for a new %s repo" % (koji.util.duration(start), tag)
return 1
def handle_help(options, session, args):
"List available commands"
usage = _("usage: %prog help [options]")

7
koji/__init__.py
View file

@ -45,6 +45,7 @@ import traceback
import urllib
import urllib2
import urlparse
import util
import xmlrpclib
from xmlrpclib import loads, Fault
import ssl.XMLRPCServerProxy
@ -912,7 +913,9 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts)
# Use the group data rather than a generated rpm
'chroot_setup_cmd': 'groupinstall build',
# don't encourage network access from the chroot
'use_host_resolv': False
'use_host_resolv': False,
# Don't let a build last more than 24 hours
'rpmbuild_timeout': 86400
}
config_opts['yum.conf'] = """[main]
@ -1383,7 +1386,7 @@ class ClientSession(object):
# raise AttributeError, "no attribute %r" % name
return VirtualMethod(self._callMethod,name)
def uploadWrapper(self, localfile, path, name=None, callback=None, blocksize=262144):
def uploadWrapper(self, localfile, path, name=None, callback=None, blocksize=1048576):
"""upload a file in chunks using the uploadFile call"""
# XXX - stick in a config or something
start=time.time()

33
koji/util.py
View file

@ -31,3 +31,36 @@ def formatChangelog(entries):
""" % (_changelogDate(entry['date']), entry['author'], entry['text'])
return result
def checkForBuilds(session, tag, builds, event):
"""Check that the builds existed in tag at the time of the event."""
for build in builds:
tagged_list = session.listTagged(tag, event=event, package=build['name'], inherit=True)
for tagged in tagged_list:
if tagged['version'] == build['version'] and tagged['release'] == build['release']:
break
else:
return False
return True
def duration(start):
"""Return the duration between start and now in MM:SS format"""
elapsed = time.time() - start
mins = int(elapsed / 60)
secs = int(elapsed % 60)
return '%s:%02i' % (mins, secs)
def printList(l):
"""Print the contents of the list comma-separated"""
if len(l) == 0:
return ''
elif len(l) == 1:
return l[0]
elif len(l) == 2:
return ' and '.join(l)
else:
ret = ', '.join(l[:-1])
ret += ', and '
ret += l[-1]
return ret

2
util/koji-gc
View file

@ -503,7 +503,7 @@ def handle_trash():
#this might happen if the build was tagged just now
print "[%i/%i] Warning: build not untagged: %s" % (i, N, nvr)
continue
age = time.time() - last['revoke_event']
age = time.time() - last['revoke_ts']
if age is not None and age < min_age:
if options.debug:
print "[%i/%i] Build untagged only recently: %s" % (i, N, nvr)