- fix koji mock-config to use sensible defaults for **opts
- add --topurl option to koji mock-config - upload in 64k chunks in incrementalUpload - best-effort to check when mock has truncated a file and started writing from 0 in BuildRoot.mock() - improvements to cmdline parsing in koji call
This commit is contained in:
Mike Bonnet
parent
f71a80d167
commit
e96a8687a0
4 changed files with 67 additions and 50 deletions
|
|
@ -212,32 +212,33 @@ def scan_mounts(topdir):
|
|||
def incrementalUpload(fname, fd, path, retries=5, logger=None):
|
||||
if not fd:
|
||||
return
|
||||
|
||||
offset = fd.tell()
|
||||
contents = fd.read()
|
||||
size = len(contents)
|
||||
if size == 0:
|
||||
return
|
||||
|
||||
data = base64.encodestring(contents)
|
||||
digest = md5.new(contents).hexdigest()
|
||||
del contents
|
||||
|
||||
tries = 0
|
||||
|
||||
while True:
|
||||
if session.uploadFile(path, fname, size, digest, offset, data):
|
||||
return
|
||||
if tries <= retries:
|
||||
tries += 1
|
||||
time.sleep(10)
|
||||
continue
|
||||
else:
|
||||
if logger:
|
||||
logger.error("Error uploading file %s to %s at offset %d" % (fname, path, offset))
|
||||
offset = fd.tell()
|
||||
contents = fd.read(65536)
|
||||
size = len(contents)
|
||||
if size == 0:
|
||||
break
|
||||
|
||||
data = base64.encodestring(contents)
|
||||
digest = md5.new(contents).hexdigest()
|
||||
del contents
|
||||
|
||||
tries = 0
|
||||
while True:
|
||||
if session.uploadFile(path, fname, size, digest, offset, data):
|
||||
break
|
||||
|
||||
if tries <= retries:
|
||||
tries += 1
|
||||
time.sleep(10)
|
||||
continue
|
||||
else:
|
||||
sys.stderr.write("Error uploading file %s to %s at offset %d" % (fname, path, offset))
|
||||
sys.stderr.write("\n")
|
||||
return
|
||||
if logger:
|
||||
logger.error("Error uploading file %s to %s at offset %d" % (fname, path, offset))
|
||||
else:
|
||||
sys.stderr.write("Error uploading file %s to %s at offset %d\n" % (fname, path, offset))
|
||||
break
|
||||
|
||||
def _parseStatus(rv, prefix):
|
||||
if isinstance(prefix, list) or isinstance(prefix, tuple):
|
||||
|
|
@ -352,8 +353,8 @@ class BuildRoot(object):
|
|||
logs = {}
|
||||
|
||||
while True:
|
||||
status = os.waitpid(pid,os.WNOHANG)
|
||||
time.sleep(1)
|
||||
status = os.waitpid(pid, os.WNOHANG)
|
||||
|
||||
try:
|
||||
results = os.listdir(resultdir)
|
||||
|
|
@ -363,19 +364,22 @@ class BuildRoot(object):
|
|||
|
||||
for fname in results:
|
||||
if fname.endswith('.log') and not logs.has_key(fname):
|
||||
logs[fname] = (None, None)
|
||||
logs[fname] = (None, None, 0)
|
||||
|
||||
for (fname, (fd, inode)) in logs.items():
|
||||
for (fname, (fd, inode, size)) in logs.items():
|
||||
try:
|
||||
fpath = os.path.join(resultdir, fname)
|
||||
stat_info = os.stat(fpath)
|
||||
if stat_info.st_ino != inode:
|
||||
if not fd or stat_info.st_ino != inode or stat_info.st_size < size:
|
||||
# either a file we haven't opened before, or mock replaced a file we had open with
|
||||
# a new file and is writing to it, but our fd is pointing to the old file
|
||||
# a new file and is writing to it, or truncated the file we're reading,
|
||||
# but our fd is pointing to the previous location in the old file
|
||||
if fd:
|
||||
self.logger.info('Rereading %s, inode: %s -> %s, size: %s -> %s' %
|
||||
(fpath, inode, stat_info.st_ino, size, stat_info.st_size))
|
||||
fd.close()
|
||||
fd = file(fpath, 'r')
|
||||
logs[fname] = (fd, stat_info.st_ino)
|
||||
logs[fname] = (fd, stat_info.st_ino, stat_info.st_size)
|
||||
except:
|
||||
self.logger.error("Error reading mock log: %s", fpath)
|
||||
self.logger.error(''.join(traceback.format_exception(*sys.exc_info())))
|
||||
|
|
@ -384,7 +388,7 @@ class BuildRoot(object):
|
|||
incrementalUpload(fname, fd, uploadpath, self.logger)
|
||||
|
||||
if status[0] != 0:
|
||||
for (fname, (fd, inode)) in logs.items():
|
||||
for (fname, (fd, inode, size)) in logs.items():
|
||||
if fd:
|
||||
fd.close()
|
||||
return status[1]
|
||||
|
|
@ -2359,9 +2363,9 @@ def get_options():
|
|||
'workdir': '/tmp/koji',
|
||||
'mockdir': '/var/lib/mock',
|
||||
'mockuser': 'kojibuilder',
|
||||
'packager': 'koji',
|
||||
'vendor': 'koji',
|
||||
'mockhost': 'linux-gnu',
|
||||
'packager': 'Koji',
|
||||
'vendor': 'Koji',
|
||||
'mockhost': 'koji-linux-gnu',
|
||||
'smtphost': 'mail@example.com',
|
||||
'from_addr': 'Koji Build System <buildsys@example.com>',
|
||||
'krb_principal': None,
|
||||
|
|
|
|||
|
|
@ -21,13 +21,13 @@
|
|||
; mockuser=kojibuilder
|
||||
|
||||
; The vendor to use in rpm headers
|
||||
; vendor=koji
|
||||
; vendor=Koji
|
||||
|
||||
; The packager to use in rpm headers
|
||||
; packager=koji
|
||||
; packager=Koji
|
||||
|
||||
; The _host string to use in mock
|
||||
; mockhost=
|
||||
; mockhost=koji-linux-gnu
|
||||
|
||||
|
||||
; The URL for the xmlrpc server
|
||||
|
|
|
|||
29
cli/koji
29
cli/koji
|
|
@ -708,14 +708,25 @@ def handle_call(options, session, args):
|
|||
name = args[0]
|
||||
non_kw = []
|
||||
kw = {}
|
||||
for arg in args[1:]:
|
||||
if arg.isdigit():
|
||||
non_kw.append(int(arg))
|
||||
elif arg.find('=') != -1:
|
||||
key, value = arg.split('=', 1)
|
||||
kw[key] = value
|
||||
|
||||
def _convarg(val):
|
||||
valmap = {'None': None,
|
||||
'True': True,
|
||||
'False': False}
|
||||
|
||||
if val.isdigit():
|
||||
return int(val)
|
||||
elif valmap.has_key(val):
|
||||
return valmap[val]
|
||||
else:
|
||||
non_kw.append(arg)
|
||||
return val
|
||||
|
||||
for arg in args[1:]:
|
||||
if arg.find('=') != -1:
|
||||
key, value = arg.split('=', 1)
|
||||
kw[key] = _convarg(value)
|
||||
else:
|
||||
non_kw.append(_convarg(arg))
|
||||
pprint.pprint(getattr(session, name).__call__(*non_kw, **kw))
|
||||
|
||||
def anon_handle_mock_config(options, session, args):
|
||||
|
|
@ -731,6 +742,8 @@ def anon_handle_mock_config(options, session, args):
|
|||
help=_("Specify mockdir"))
|
||||
parser.add_option("--topdir", metavar="DIR",
|
||||
help=_("Specify topdir"))
|
||||
parser.add_option("--topurl", metavar="URL",
|
||||
help=_("url under which Koji files are accessible"))
|
||||
parser.add_option("--distribution", default="Koji Testing",
|
||||
help=_("Change the distribution macro"))
|
||||
parser.add_option("-o", metavar="FILE", dest="ofile", help=_("Output to a file"))
|
||||
|
|
@ -742,7 +755,7 @@ def anon_handle_mock_config(options, session, args):
|
|||
name = args[0]
|
||||
arch = None
|
||||
opts = {}
|
||||
for k in ('topdir', 'distribution', 'mockdir'):
|
||||
for k in ('topdir', 'topurl', 'distribution', 'mockdir'):
|
||||
if hasattr(options, k):
|
||||
opts[k] = getattr(options, k)
|
||||
if options.buildroot:
|
||||
|
|
|
|||
|
|
@ -885,7 +885,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts)
|
|||
repodir = pathinfo.repo(repoid,tag_name)
|
||||
url = "%s/%s" % (repodir,arch)
|
||||
else:
|
||||
pathinfo = PathInfo(topdir=opts.get('topdir'))
|
||||
pathinfo = PathInfo(topdir=opts.get('topdir', '/mnt/koji'))
|
||||
repodir = pathinfo.repo(repoid,tag_name)
|
||||
url = "file://%s/%s" % (repodir,arch)
|
||||
if managed:
|
||||
|
|
@ -933,14 +933,14 @@ baseurl=%(url)s
|
|||
macros = {
|
||||
'_topdir' : '%s/build' % config_opts['chroothome'],
|
||||
'_rpmfilename' : '%%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm',
|
||||
'vendor' : opts['vendor'],
|
||||
'packager' : opts['packager'],
|
||||
'vendor' : opts.get('vendor', 'Koji'),
|
||||
'packager' : opts.get('packager', 'Koji'),
|
||||
'_host_cpu' : arch,
|
||||
'_host': '%s-%s' % (arch, opts['mockhost']),
|
||||
'_host': '%s-%s' % (arch, opts.get('mockhost', 'koji-linux-gnu')),
|
||||
#TODO - track some of these in the db instead?
|
||||
}
|
||||
if opts.has_key('distribution'):
|
||||
macros['distribution'] = opts['distribution']
|
||||
macros['distribution'] = opts.get('distribution')
|
||||
config_opts['macros'] = '\n'.join(["%%%s %s" % (k, v) for k,v in macros.iteritems()])
|
||||
|
||||
parts = ["""#!/usr/bin/python -tt
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue