use only relative paths + volumes in downloadTaskOutput
This commit is contained in:
Tomas Kopecek
parent
d40d934d6a
commit
4ef0bc2051
5 changed files with 58 additions and 49 deletions
83
cli/koji
83
cli/koji
|
|
@ -508,24 +508,28 @@ def watch_logs(session, tasklist, opts):
|
|||
if _isDone(session, task_id):
|
||||
tasklist.remove(task_id)
|
||||
|
||||
output = session.listTaskOutput(task_id)
|
||||
output = session.listTaskOutput(task_id, all_volumes=True)
|
||||
# convert to list of (file, volume)
|
||||
files = []
|
||||
for filename, volumes in output.iteritems():
|
||||
files += [(filename, volume) for volume in volumes]
|
||||
|
||||
if opts.log:
|
||||
logs = [filename for filename in output if filename == opts.log]
|
||||
logs = [file_volume for file_volume in files if file_volume[0] == opts.log]
|
||||
else:
|
||||
logs = [filename for filename in output if filename.endswith('.log')]
|
||||
logs = [file_volume for file_volume in files if file_volume[0].endswith('log')]
|
||||
|
||||
taskoffsets = offsets[task_id]
|
||||
for log in logs:
|
||||
for log, volume in logs:
|
||||
contents = 'placeholder'
|
||||
while contents:
|
||||
if log not in taskoffsets:
|
||||
taskoffsets[log] = 0
|
||||
if (log, volume) not in taskoffsets:
|
||||
taskoffsets[(log, volume)] = 0
|
||||
|
||||
contents = session.downloadTaskOutput(task_id, log, taskoffsets[log], 16384)
|
||||
taskoffsets[log] += len(contents)
|
||||
contents = session.downloadTaskOutput(task_id, log, taskoffsets[(log, volume)], 16384, volume=volume)
|
||||
taskoffsets[(log, volume)] += len(contents)
|
||||
if contents:
|
||||
currlog = "%d:%s:" % (task_id, log)
|
||||
currlog = "%d:%s:%s:" % (task_id, volume, log)
|
||||
if currlog != lastlog:
|
||||
if lastlog:
|
||||
sys.stdout.write("\n")
|
||||
|
|
@ -6761,9 +6765,13 @@ def anon_handle_download_logs(options, session, args):
|
|||
sys.stdout.write("Writing: %s\n" % full_filename)
|
||||
file(full_filename, 'w').write(content)
|
||||
|
||||
def download_log(task_log_dir, task_id, filename, blocksize=102400):
|
||||
#Create directories only if there is any log file to write to
|
||||
full_filename = os.path.normpath(os.path.join(task_log_dir, filename))
|
||||
def download_log(task_log_dir, task_id, filename, blocksize=102400, volume=None):
|
||||
# Create directories only if there is any log file to write to
|
||||
# For each non-default volume create special sub-directory
|
||||
if volume not in (None, 'DEFAULT'):
|
||||
full_filename = os.path.normpath(os.path.join(task_log_dir, volume, filename))
|
||||
else:
|
||||
full_filename = os.path.normpath(os.path.join(task_log_dir, filename))
|
||||
koji.ensuredir(os.path.dirname(full_filename))
|
||||
contents = 'IGNORE ME!'
|
||||
if suboptions.cont and os.path.exists(full_filename):
|
||||
|
|
@ -6776,7 +6784,7 @@ def anon_handle_download_logs(options, session, args):
|
|||
offset = 0
|
||||
try:
|
||||
while contents:
|
||||
contents = session.downloadTaskOutput(task_id, filename, offset, blocksize)
|
||||
contents = session.downloadTaskOutput(task_id, filename, offset=offset, size=blocksize, volume=volume)
|
||||
offset += len(contents)
|
||||
if contents:
|
||||
fd.write(contents)
|
||||
|
|
@ -6788,14 +6796,14 @@ def anon_handle_download_logs(options, session, args):
|
|||
task_info = session.getTaskInfo(task_id)
|
||||
if task_info is None:
|
||||
error(_("No such task id: %i" % task_id))
|
||||
files = session.listTaskOutput(task_id)
|
||||
logs = []
|
||||
files = session.listTaskOutput(task_id, all_volumes=True)
|
||||
logs = [] # list of tuples (filename, volume)
|
||||
for filename in files:
|
||||
if not filename.endswith(".log"):
|
||||
continue
|
||||
if match and not koji.util.multi_fnmatch(filename, match):
|
||||
continue
|
||||
logs.append(filename)
|
||||
logs += [(filename, volume) for volume in files[filename]]
|
||||
|
||||
task_log_dir = os.path.join(parent_dir,
|
||||
"%s-%s" % (task_info["arch"], task_id))
|
||||
|
|
@ -6809,8 +6817,8 @@ def anon_handle_download_logs(options, session, args):
|
|||
elif state not in ['CLOSED', 'CANCELED']:
|
||||
sys.stderr.write(_("Warning: task %s is %s\n") % (task_id, state))
|
||||
|
||||
for log_filename in logs:
|
||||
download_log(task_log_dir, task_id, log_filename)
|
||||
for log_filename, log_volume in logs:
|
||||
download_log(task_log_dir, task_id, log_filename, volume=log_volume)
|
||||
count += 1
|
||||
|
||||
if count == 0 and not recurse:
|
||||
|
|
@ -6876,25 +6884,27 @@ def anon_handle_download_task(options, session, args):
|
|||
downloads = []
|
||||
|
||||
for task in downloadable_tasks:
|
||||
files = session.listTaskOutput(task["id"])
|
||||
files = session.listTaskOutput(task["id"], all_volumes=True)
|
||||
for filename in files:
|
||||
if filename.endswith(".log") and suboptions.logs:
|
||||
# rename logs, they would conflict
|
||||
new_filename = "%s.%s.log" % (filename.rstrip(".log"), task["arch"])
|
||||
downloads.append((task, filename, new_filename))
|
||||
continue
|
||||
for volume in files[filename]:
|
||||
# rename logs, they would conflict
|
||||
new_filename = "%s.%s.log" % (filename.rstrip(".log"), task["arch"])
|
||||
downloads.append((task, filename, volume, new_filename))
|
||||
continue
|
||||
|
||||
if filename.endswith(".rpm"):
|
||||
filearch = filename.split(".")[-2]
|
||||
if len(suboptions.arches) == 0 or filearch in suboptions.arches:
|
||||
downloads.append((task, filename, filename))
|
||||
continue
|
||||
for volume in files[filename]:
|
||||
filearch = filename.split(".")[-2]
|
||||
if len(suboptions.arches) == 0 or filearch in suboptions.arches:
|
||||
downloads.append((task, filename, volume, filename))
|
||||
continue
|
||||
|
||||
if len(downloads) == 0:
|
||||
error(_("No files for download found."))
|
||||
|
||||
required_tasks = {}
|
||||
for (task, nop, nop) in downloads:
|
||||
for (task, nop, nop, nop) in downloads:
|
||||
if task["id"] not in required_tasks:
|
||||
required_tasks[task["id"]] = task
|
||||
|
||||
|
|
@ -6908,11 +6918,14 @@ def anon_handle_download_task(options, session, args):
|
|||
# perform the download
|
||||
|
||||
number = 0
|
||||
for (task, filename, new_filename) in downloads:
|
||||
for (task, filename, volume, new_filename) in downloads:
|
||||
number += 1
|
||||
if volume not in (None, 'DEFAULT'):
|
||||
koji.ensuredir(volume)
|
||||
new_filename = os.path.join(volume, new_filename)
|
||||
print(_("Downloading [%d/%d]: %s") % (number, len(downloads), new_filename))
|
||||
output_file = open(new_filename, "wb")
|
||||
output_file.write(session.downloadTaskOutput(task["id"], filename))
|
||||
output_file.write(session.downloadTaskOutput(task["id"], filename, volume=volume))
|
||||
output_file.close()
|
||||
|
||||
def anon_handle_wait_repo(options, session, args):
|
||||
|
|
@ -7168,11 +7181,11 @@ def handle_runroot(options, session, args):
|
|||
print("User interrupt: canceling runroot task")
|
||||
session.cancelTask(task_id)
|
||||
raise
|
||||
output = None
|
||||
if "runroot.log" in session.listTaskOutput(task_id):
|
||||
output = session.downloadTaskOutput(task_id, "runroot.log")
|
||||
if output:
|
||||
sys.stdout.write(output)
|
||||
output = session.listTaskOutput(task_id, all_volumes=True)
|
||||
if 'runroot.log' in output:
|
||||
for volume in output['runroot.log']:
|
||||
log = session.downloadTaskOutput(task_id, 'runroot.log', volume=volume)
|
||||
sys.stdout.write(log)
|
||||
info = session.getTaskInfo(task_id)
|
||||
if info is None:
|
||||
sys.exit(1)
|
||||
|
|
|
|||
|
|
@ -8857,16 +8857,12 @@ class RootExports(object):
|
|||
os.close(fd)
|
||||
|
||||
|
||||
def downloadTaskOutput(self, taskID, fileName, offset=0, size=-1):
|
||||
def downloadTaskOutput(self, taskID, fileName, offset=0, size=-1, volume=None):
|
||||
"""Download the file with the given name, generated by the task with the
|
||||
given ID."""
|
||||
if '..' in fileName:
|
||||
raise koji.GenericError('Invalid file name: %s' % fileName)
|
||||
if not fileName.startswith('/'):
|
||||
filePath = '%s/%s/%s' % (koji.pathinfo.work(), koji.pathinfo.taskrelpath(taskID), fileName)
|
||||
else:
|
||||
filePath = fileName
|
||||
assert(koji.pathinfo.taskrelpath(taskID) in filePath)
|
||||
filePath = '%s/%s/%s' % (koji.pathinfo.work(volume), koji.pathinfo.taskrelpath(taskID), fileName)
|
||||
filePath = os.path.normpath(filePath)
|
||||
if not os.path.isfile(filePath):
|
||||
raise koji.GenericError('no file "%s" output by task %i' % (fileName, taskID))
|
||||
|
|
|
|||
|
|
@ -2611,7 +2611,7 @@ class ClientSession(object):
|
|||
callback(ofs, totalsize, size, t1, t2)
|
||||
fo.close()
|
||||
|
||||
def downloadTaskOutput(self, taskID, fileName, offset=0, size=-1):
|
||||
def downloadTaskOutput(self, taskID, fileName, offset=0, size=-1, volume=None):
|
||||
"""Download the file with the given name, generated by the task with the
|
||||
given ID.
|
||||
|
||||
|
|
@ -2619,7 +2619,7 @@ class ClientSession(object):
|
|||
"""
|
||||
if self.multicall:
|
||||
raise GenericError('downloadTaskOutput() may not be called during a multicall')
|
||||
result = self.callMethod('downloadTaskOutput', taskID, fileName, offset, size)
|
||||
result = self.callMethod('downloadTaskOutput', taskID, fileName, offset=offset, size=size, volume=volume)
|
||||
return base64.decodestring(result)
|
||||
|
||||
class DBHandler(logging.Handler):
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ class TestListCommands(unittest.TestCase):
|
|||
# Mock out the xmlrpc server
|
||||
self.session.getTaskInfo.return_value = {'state': 1}
|
||||
self.session.downloadTaskOutput.return_value = 'task output'
|
||||
self.session.listTaskOutput.return_value = ['runroot.log']
|
||||
self.session.listTaskOutput.return_value = {'runroot.log': ['DEFAULT']}
|
||||
self.session.runroot.return_value = 1
|
||||
|
||||
# Run it and check immediate output
|
||||
|
|
@ -54,9 +54,9 @@ class TestListCommands(unittest.TestCase):
|
|||
|
||||
# Finally, assert that things were called as we expected.
|
||||
self.session.getTaskInfo.assert_called_once_with(1)
|
||||
self.session.listTaskOutput.assert_called_once_with(1)
|
||||
self.session.listTaskOutput.assert_called_once_with(1, all_volumes=True)
|
||||
self.session.downloadTaskOutput.assert_called_once_with(
|
||||
1, 'runroot.log')
|
||||
1, 'runroot.log', volume='DEFAULT')
|
||||
self.session.runroot.assert_called_once_with(
|
||||
tag, arch, command, repo_id=mock.ANY, weight=mock.ANY,
|
||||
mounts=mock.ANY, packages=mock.ANY, skip_setarch=mock.ANY,
|
||||
|
|
|
|||
|
|
@ -767,10 +767,10 @@ def getfile(environ, taskID, name, volume='DEFAULT', offset=None, size=None):
|
|||
size = file_size - offset
|
||||
|
||||
#environ['koji.headers'].append(['Content-Length', str(size)])
|
||||
return _chunk_file(server, environ, taskID, name, offset, size)
|
||||
return _chunk_file(server, environ, taskID, name, offset, size, volume)
|
||||
|
||||
|
||||
def _chunk_file(server, environ, taskID, name, offset, size):
|
||||
def _chunk_file(server, environ, taskID, name, offset, size, volume):
|
||||
remaining = size
|
||||
encode_int = koji.encode_int
|
||||
while True:
|
||||
|
|
@ -779,7 +779,7 @@ def _chunk_file(server, environ, taskID, name, offset, size):
|
|||
chunk_size = 1048576
|
||||
if remaining < chunk_size:
|
||||
chunk_size = remaining
|
||||
content = server.downloadTaskOutput(taskID, name, offset=encode_int(offset), size=chunk_size)
|
||||
content = server.downloadTaskOutput(taskID, name, offset=encode_int(offset), size=chunk_size, volume=volume)
|
||||
if not content:
|
||||
break
|
||||
yield content
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue