diff --git a/hub/kojihub.py b/hub/kojihub.py index 9383ada3..0c10ad50 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -4260,7 +4260,7 @@ def get_archive_file(archive_id, filename): #otherwise return None -def list_task_output(taskID, stat=False): +def list_task_output(taskID, stat=False, all_volumes=False): """List the files generated by the task with the given ID. This will usually include one or more RPMs, and one or more log files. If the task did not generate any files, or the output directory @@ -4270,29 +4270,60 @@ def list_task_output(taskID, stat=False): is a map containing the values of the st_* attributes returned by os.stat(). - It goes through all available volumes""" - if stat: + If all_volumes is set, results are extended to deal with files in same + relative paths on different volumes. + + With all_volumes=True, stat=False, return a map of filename -> list_of_volumes, + {'stdout.log': ['DEFAULT']} + + With all_volumes=True, stat=True, return a map of + filename -> map_of_volumes -> stat_info, + {'stdout.log': + {'DEFAULT': { + { + 'st_atime': 1488902587.2141163, + 'st_ctime': 1488902588.2281106, + 'st_mtime': 1488902588.2281106, + 'st_size': '526' + } + } + } + """ + if stat or all_volumes: result = {} else: result = [] - for vol_info in list_volumes(): - taskDir = '%s/%s' % (koji.pathinfo.work(volume=vol_info['name']), koji.pathinfo.taskrelpath(taskID)) + + if all_volumes: + volumes = [x['name'] for x in list_volumes()] + else: + volumes = ['DEFAULT'] + + for volume in volumes: + taskDir = '%s/%s' % (koji.pathinfo.work(volume=volume), koji.pathinfo.taskrelpath(taskID)) if not os.path.isdir(taskDir): continue for path, dirs, files in os.walk(taskDir): for filename in files: - filename = os.path.join(path, filename) + relpath = path[len(taskDir) + 1:] + relfilename = os.path.join(relpath, filename) if stat: - stat_info = os.stat(filename) + stat_info = os.stat(os.path.join(path, filename)) stat_map = {} for attr in dir(stat_info): if attr == 'st_size': stat_map[attr] = str(getattr(stat_info, attr)) elif attr in ('st_atime', 'st_mtime', 'st_ctime'): stat_map[attr] = getattr(stat_info, attr) - result[filename] = stat_map + if all_volumes: + result.setdefault(relfilename, {})[volume] = stat_map + else: + result[relfilename] = stat_map else: - result.append(filename) + if all_volumes: + result.setdefault(relfilename, []).append(volume) + else: + result.append(relfilename) return result def _fetchMulti(query, values): diff --git a/tests/test_hub/test_list_task_output.py b/tests/test_hub/test_list_task_output.py new file mode 100644 index 00000000..0b27af1a --- /dev/null +++ b/tests/test_hub/test_list_task_output.py @@ -0,0 +1,88 @@ +import unittest +import mock + +import kojihub + +class TestListTaskOutput(unittest.TestCase): + @mock.patch('os.path.isdir') + @mock.patch('os.walk') + def test_empty(self, walk, isdir): + isdir.return_value = True + walk.return_value = [] + result = kojihub.list_task_output(1) + self.assertEqual(result, []) + + @mock.patch('os.path.isdir') + @mock.patch('os.walk') + def test_simple(self, walk, isdir): + isdir.return_value = True + walk.return_value = (('dir', [], ['file']),) + result = kojihub.list_task_output(1) + self.assertEqual(result, ['file']) + + @mock.patch('os.stat') + @mock.patch('os.path.isdir') + @mock.patch('os.walk') + def test_simple_stat(self, walk, isdir, stat): + isdir.return_value = True + walk.return_value = (('dir', [], ['file']),) + st_mock = mock.MagicMock() + st_mock.st_size = 123 + st_mock.st_atime = 345 + st_mock.st_mtime = 678 + st_mock.st_ctime = 901 + stat.return_value = st_mock + result = kojihub.list_task_output(1, stat=True) + + self.assertEqual(result, { + 'file': { + 'st_size': '123', + 'st_atime': 345, + 'st_mtime': 678, + 'st_ctime': 901, + } + }) + + @mock.patch('kojihub.list_volumes') + @mock.patch('os.stat') + @mock.patch('os.path.isdir') + @mock.patch('os.walk') + def test_volumes(self, walk, isdir, stat, list_volumes): + isdir.return_value = True + walk.return_value = (('dir', [], ['file']),) + st_mock = mock.MagicMock() + st_mock.st_size = 123 + st_mock.st_atime = 345 + st_mock.st_mtime = 678 + st_mock.st_ctime = 901 + stat.return_value = st_mock + list_volumes.return_value = [{'name': 'DEFAULT'}] + result = kojihub.list_task_output(1, all_volumes=True) + self.assertEqual(result, {'file': ['DEFAULT']}) + + @mock.patch('kojihub.list_volumes') + @mock.patch('os.stat') + @mock.patch('os.path.isdir') + @mock.patch('os.walk') + def test_volumes_stat(self, walk, isdir, stat, list_volumes): + isdir.return_value = True + walk.return_value = (('dir', [], ['file']),) + st_mock = mock.MagicMock() + st_mock.st_size = 123 + st_mock.st_atime = 345 + st_mock.st_mtime = 678 + st_mock.st_ctime = 901 + stat.return_value = st_mock + list_volumes.return_value = [{'name': 'DEFAULT'}] + result = kojihub.list_task_output(1, stat=True, all_volumes=True) + + self.assertEqual(result, { + 'file': { + 'DEFAULT': { + 'st_size': '123', + 'st_atime': 345, + 'st_mtime': 678, + 'st_ctime': 901, + } + } + }) diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py index 17b25c74..88e1d6b2 100644 --- a/www/kojiweb/index.py +++ b/www/kojiweb/index.py @@ -672,18 +672,19 @@ def taskinfo(environ, taskID): values['full_result_text'] = full_result_text values['abbr_result_text'] = abbr_result_text - output = server.listTaskOutput(task['id']) - output = [p[len(koji.pathinfo.topdir):] for p in output] - output.sort(_sortByExtAndName) - values['output'] = output + topurl = environ['koji.options']['KojiFilesURL'] + pathinfo = koji.PathInfo(topdir=topurl) + values['pathinfo'] = pathinfo + + paths = [] # (volume, relpath) tuples + for relname, volumes in server.listTaskOutput(task['id'], all_volumes=True).iteritems(): + paths += [(volume, relname) for volume in volumes] + values['output'] = sorted(paths, cmp = _sortByExtAndName) if environ['koji.currentUser']: values['perms'] = server.getUserPerms(environ['koji.currentUser']['id']) else: values['perms'] = [] - topurl = environ['koji.options']['KojiFilesURL'] - values['pathinfo'] = koji.PathInfo(topdir=topurl) - return _genHTML(environ, 'taskinfo.chtml') def taskstatus(environ, taskID): @@ -693,11 +694,11 @@ def taskstatus(environ, taskID): task = server.getTaskInfo(taskID) if not task: return '' - files = server.listTaskOutput(taskID, stat=True) + files = server.listTaskOutput(taskID, stat=True, all_volumes=True) output = '%i:%s\n' % (task['id'], koji.TASK_STATES[task['state']]) - for filename, file_stats in files.items(): - output += '%s:%s\n' % (filename, file_stats['st_size']) - + for filename, volumes_data in files.iteritems(): + for volume, file_stats in volumes_data.iteritems(): + output += '%s:%s:%s\n' % (volume, filename, file_stats['st_size']) return output def resubmittask(environ, taskID): @@ -717,19 +718,19 @@ def canceltask(environ, taskID): _redirect(environ, 'taskinfo?taskID=%i' % taskID) def _sortByExtAndName(a, b): - """Sort two filenames, first by extension, and then by name.""" - aRoot, aExt = os.path.splitext(os.path.basename(a)) - bRoot, bExt = os.path.splitext(os.path.basename(b)) + """Sort two filename tuples, first by extension, and then by name.""" + aRoot, aExt = os.path.splitext(os.path.basename(a[1])) + bRoot, bExt = os.path.splitext(os.path.basename(b[1])) return cmp(aExt, bExt) or cmp(aRoot, bRoot) -def getfile(environ, taskID, name, offset=None, size=None): +def getfile(environ, taskID, name, volume='DEFAULT', offset=None, size=None): server = _getServer(environ) taskID = int(taskID) - output = server.listTaskOutput(taskID, stat=True) - name = os.path.join(koji.pathinfo.topdir, name[1:]) - file_info = output.get(name) - if not file_info: + output = server.listTaskOutput(taskID, stat=True, all_volumes=True) + try: + file_info = output[name][volume] + except KeyError: raise koji.GenericError('no file "%s" output by task %i' % (name, taskID)) mime_guess = mimetypes.guess_type(name, strict=False)[0] diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 9ec91013..b8bb7971 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -412,10 +412,10 @@ $value Output - #for $filename in $output - $filename
+ #for $volume, $filename in $output + $filename
#if $filename.endswith('.log') - (tail) + (tail) #end if
#end for diff --git a/www/static/js/watchlogs.js b/www/static/js/watchlogs.js index 2f0cdcd2..2ec1b6ac 100644 --- a/www/static/js/watchlogs.js +++ b/www/static/js/watchlogs.js @@ -51,8 +51,8 @@ function handleStatus(event) { var logs = {}; for (var i = 1; i < lines.length; i++) { data = lines[i].split(":"); - var filename = data[0]; - var filesize = parseInt(data[1]); + var filename = data[0] + ":" + data[1]; + var filesize = parseInt(data[2]); if (filename.indexOf(".log") != -1) { logs[filename] = filesize; } @@ -140,8 +140,11 @@ function outputLog() { chunkSize = currentSize - currentOffset; } var req = new XMLHttpRequest(); - req.open("GET", baseURL + "/getfile?taskID=" + currentTaskID + "&name=" + currentLog + - "&offset=" + currentOffset + "&size=" + chunkSize, true); + var data = currentLog.split(':'); + var volume = data[0]; + var filename = data[1]; + req.open("GET", baseURL + "/getfile?taskID=" + currentTaskID + "&name=" + filename + + "&volume=" + volume + "&offset=" + currentOffset + "&size=" + chunkSize, true); req.onreadystatechange = handleLog; req.send(null); if (headerElement != null) {