particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Allow uploading files to non-default volumes

Browse source
This commit is contained in:
Tomas Kopecek 2016-11-14 13:15:36 +01:00
parent 439f2fef1b
commit 323987e376
6 changed files with 59 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

59
hub/kojihub.py
View file

@ -4268,29 +4268,31 @@ def list_task_output(taskID, stat=False):
If stat is True, return a map of filename -> stat_info where stat_info
is a map containing the values of the st_* attributes returned by
os.stat()."""
taskDir = '%s/%s' % (koji.pathinfo.work(), koji.pathinfo.taskrelpath(taskID))
os.stat().
It goes through all available volumes"""
if stat:
result = {}
else:
result = []
if not os.path.isdir(taskDir):
return result
for path, dirs, files in os.walk(taskDir):
relpath = path[len(taskDir) + 1:]
for filename in files:
relfilename = os.path.join(relpath, filename)
if stat:
stat_info = os.stat(os.path.join(path, filename))
stat_map = {}
for attr in dir(stat_info):
if attr == 'st_size':
stat_map[attr] = str(getattr(stat_info, attr))
elif attr in ('st_atime', 'st_mtime', 'st_ctime'):
stat_map[attr] = getattr(stat_info, attr)
result[relfilename] = stat_map
else:
result.append(relfilename)
for vol_info in list_volumes():
taskDir = '%s/%s' % (koji.pathinfo.work(volume=vol_info['name']), koji.pathinfo.taskrelpath(taskID))
if not os.path.isdir(taskDir):
continue
for path, dirs, files in os.walk(taskDir):
for filename in files:
filename = os.path.join(path, filename)
if stat:
stat_info = os.stat(filename)
stat_map = {}
for attr in dir(stat_info):
if attr == 'st_size':
stat_map[attr] = str(getattr(stat_info, attr))
elif attr in ('st_atime', 'st_mtime', 'st_ctime'):
stat_map[attr] = getattr(stat_info, attr)
result[filename] = stat_map
else:
result.append(filename)
return result
def _fetchMulti(query, values):
@ -8683,7 +8685,7 @@ class RootExports(object):
context.session.assertPerm('admin')
return make_task(*args, **opts)
def uploadFile(self, path, name, size, md5sum, offset, data):
def uploadFile(self, path, name, size, md5sum, offset, data, volume=None):
#path: the relative path to upload to
#name: the name of the file
#size: size of contents (bytes)
@ -8715,7 +8717,7 @@ class RootExports(object):
if verify is not None:
if digest != sum_cls(contents).hexdigest():
return False
fn = get_upload_path(path, name, create=True)
fn = get_upload_path(path, name, create=True, volume=volume)
try:
st = os.lstat(fn)
except OSError, e:
@ -8829,7 +8831,11 @@ class RootExports(object):
given ID."""
if '..' in fileName:
raise koji.GenericError('Invalid file name: %s' % fileName)
filePath = '%s/%s/%s' % (koji.pathinfo.work(), koji.pathinfo.taskrelpath(taskID), fileName)
if not fileName.startswith('/'):
filePath = '%s/%s/%s' % (koji.pathinfo.work(), koji.pathinfo.taskrelpath(taskID), fileName)
else:
filePath = fileName
assert(koji.pathinfo.taskrelpath(taskID) in filePath)
filePath = os.path.normpath(filePath)
if not os.path.isfile(filePath):
raise koji.GenericError('no file "%s" output by task %i' % (fileName, taskID))
@ -12240,7 +12246,12 @@ class HostExports(object):
return host.isEnabled()
def get_upload_path(reldir, name, create=False):
def get_upload_path(reldir, name, create=False, volume=None):
if volume is not None:
volinfo = lookup_name('volume', volume, strict=True)
pathinfo = koji.PathInfo(topdir=koji.pathinfo.volumedir(volinfo['name']))
else:
pathinfo = koji.pathinfo
orig_reldir = reldir
orig_name = name
# lots of sanity checks
@ -12264,7 +12275,7 @@ def get_upload_path(reldir, name, create=False):
host.verify()
Task(task_id).assertHost(host.id)
check_user = False
udir = os.path.join(koji.pathinfo.work(), reldir)
udir = os.path.join(pathinfo.work(), reldir)
if create:
koji.ensuredir(udir)
if check_user:

26
koji/__init__.py
View file

@ -1823,24 +1823,24 @@ class PathInfo(object):
"""Return the relative path for the task work directory"""
return "tasks/%s/%s" % (task_id % 10000, task_id)
def work(self):
def work(self, volume=None):
"""Return the work dir"""
return self.topdir + '/work'
return self.volumedir(volume) + '/work'
def tmpdir(self):
def tmpdir(self, volume=None):
"""Return a path to a unique directory under work()/tmp/"""
tmp = None
while tmp is None or os.path.exists(tmp):
tmp = self.work() + '/tmp/' + ''.join([random.choice(self.ASCII_CHARS) for dummy in '123456'])
tmp = self.work(volume) + '/tmp/' + ''.join([random.choice(self.ASCII_CHARS) for dummy in '123456'])
return tmp
def scratch(self):
"""Return the main scratch dir"""
return self.topdir + '/scratch'
def task(self, task_id):
def task(self, task_id, volume=None):
"""Return the output directory for the task with the given id"""
return self.work() + '/' + self.taskrelpath(task_id)
return self.work(volume=volume) + '/' + self.taskrelpath(task_id)
pathinfo = PathInfo()
@ -2451,7 +2451,7 @@ class ClientSession(object):
# raise AttributeError("no attribute %r" % name)
return VirtualMethod(self._callMethod, name)
def fastUpload(self, localfile, path, name=None, callback=None, blocksize=None, overwrite=False):
def fastUpload(self, localfile, path, name=None, callback=None, blocksize=None, overwrite=False, volume=None):
if blocksize is None:
blocksize = self.opts.get('upload_blocksize', 1048576)
@ -2476,7 +2476,7 @@ class ClientSession(object):
if not chunk and not first_cycle:
break
first_cycle = False
result = self._callMethod('rawUpload', (chunk, ofs, path, name), {'overwrite':overwrite})
result = self._callMethod('rawUpload', (chunk, ofs, path, name), {'overwrite':overwrite, 'volume': volume})
if self.retries > 1:
problems = True
hexdigest = util.adler32_constructor(chunk).hexdigest()
@ -2509,7 +2509,7 @@ class ClientSession(object):
% (path, name, result['hexdigest'], full_chksum.hexdigest()))
self.logger.debug("Fast upload: %s complete. %i bytes in %.1f seconds", localfile, size, t2)
def _prepUpload(self, chunk, offset, path, name, verify="adler32", overwrite=False):
def _prepUpload(self, chunk, offset, path, name, verify="adler32", overwrite=False, volume=None):
"""prep a rawUpload call"""
if not self.logged_in:
raise ActionNotAllowed("you must be logged in to upload")
@ -2532,13 +2532,13 @@ class ClientSession(object):
request = chunk
return handler, headers, request
def uploadWrapper(self, localfile, path, name=None, callback=None, blocksize=None, overwrite=True):
def uploadWrapper(self, localfile, path, name=None, callback=None, blocksize=None, overwrite=True, volume=None):
"""upload a file in chunks using the uploadFile call"""
if blocksize is None:
blocksize = self.opts.get('upload_blocksize', 1048576)
if self.opts.get('use_fast_upload'):
self.fastUpload(localfile, path, name, callback, blocksize, overwrite)
self.fastUpload(localfile, path, name, callback, blocksize, overwrite, volume=volume)
return
if name is None:
name = os.path.basename(localfile)
@ -2551,7 +2551,7 @@ class ClientSession(object):
except GenericError:
pass
else:
self.fastUpload(localfile, path, name, callback, blocksize, overwrite)
self.fastUpload(localfile, path, name, callback, blocksize, overwrite, volume=volume)
return
start = time.time()
@ -2584,7 +2584,7 @@ class ClientSession(object):
while True:
if debug:
self.logger.debug("uploadFile(%r,%r,%r,%r,%r,...)" %(path, name, sz, digest, offset))
if self.callMethod('uploadFile', path, name, encode_int(sz), digest, encode_int(offset), data):
if self.callMethod('uploadFile', path, name, encode_int(sz), digest, encode_int(offset), data, volume=volume):
break
if tries <= retries:
tries += 1

8
koji/tasks.py
View file

@ -262,7 +262,7 @@ class BaseTaskHandler(object):
def getUploadDir(self):
return koji.pathinfo.taskrelpath(self.id)
def uploadFile(self, filename, relPath=None, remoteName=None):
def uploadFile(self, filename, relPath=None, remoteName=None, volume=None):
"""Upload the file with the given name to the task output directory
on the hub."""
uploadPath = self.getUploadDir()
@ -271,9 +271,9 @@ class BaseTaskHandler(object):
uploadPath += '/' + relPath
# Only upload files with content
if os.path.isfile(filename) and os.stat(filename).st_size > 0:
self.session.uploadWrapper(filename, uploadPath, remoteName)
self.session.uploadWrapper(filename, uploadPath, remoteName, volume=volume)
def uploadTree(self, dirpath, flatten=False):
def uploadTree(self, dirpath, flatten=False, volume=None):
"""Upload the directory tree at dirpath to the task directory on the
hub, preserving the directory structure"""
dirpath = dirpath.rstrip('/')
@ -283,7 +283,7 @@ class BaseTaskHandler(object):
else:
relpath = path[len(dirpath) + 1:]
for filename in files:
self.uploadFile(os.path.join(path, filename), relpath)
self.uploadFile(os.path.join(path, filename), relpath, volume=volume)
def chownTree(self, dirpath, uid, gid):
"""chown the given path and all files and directories under

4
tests/test_tasks.py
View file

@ -380,7 +380,7 @@ class TasksTestCase(TestCase):
obj = TestTask(123, 'some_method', ['random_arg'], None, None, temp_path)
obj.session = Mock()
self.assertEquals(obj.uploadFile(temp_file), None)
obj.session.uploadWrapper.assert_called_once_with(temp_file, 'tasks/123/123', None)
obj.session.uploadWrapper.assert_called_once_with(temp_file, 'tasks/123/123', None, volume=None)
# This patch removes the dependence on getUploadDir functioning
@patch('{0}.TestTask.getUploadDir'.format(__name__), return_value='tasks/123/123')
@ -421,7 +421,7 @@ class TasksTestCase(TestCase):
obj.uploadFile = Mock()
obj.uploadFile.return_value = None
self.assertEquals(obj.uploadTree(temp_path), None)
obj.uploadFile.assert_has_calls([call(dummy_file, ''), call(dummy_file2, 'some_directory')])
obj.uploadFile.assert_has_calls([call(dummy_file, '', volume=None), call(dummy_file2, 'some_directory', volume=None)])
@patch('os.lchown', return_value=None)
def test_BaseTaskHandler_chownTree(self, mock_lchown):

6
www/kojiweb/index.py
View file

@ -673,6 +673,7 @@ def taskinfo(environ, taskID):
values['abbr_result_text'] = abbr_result_text
output = server.listTaskOutput(task['id'])
output = [p[len(koji.pathinfo.topdir):] for p in output]
output.sort(_sortByExtAndName)
values['output'] = output
if environ['koji.currentUser']:
@ -717,8 +718,8 @@ def canceltask(environ, taskID):
def _sortByExtAndName(a, b):
"""Sort two filenames, first by extension, and then by name."""
aRoot, aExt = os.path.splitext(a)
bRoot, bExt = os.path.splitext(b)
aRoot, aExt = os.path.splitext(os.path.basename(a))
bRoot, bExt = os.path.splitext(os.path.basename(b))
return cmp(aExt, bExt) or cmp(aRoot, bRoot)
def getfile(environ, taskID, name, offset=None, size=None):
@ -726,6 +727,7 @@ def getfile(environ, taskID, name, offset=None, size=None):
taskID = int(taskID)
output = server.listTaskOutput(taskID, stat=True)
name = os.path.join(koji.pathinfo.topdir, name[1:])
file_info = output.get(name)
if not file_info:
raise koji.GenericError('no file "%s" output by task %i' % (name, taskID))

2
www/kojiweb/taskinfo.chtml
View file

@ -413,7 +413,7 @@ $value
<th>Output</th>
<td>
#for $filename in $output
<a href="$pathinfo.task($task.id)/$urllib.quote($filename)">$filename</a>
<a href="$pathinfo.topdir$urllib.quote($filename)">$filename</a><br/>
#if $filename.endswith('.log')
(<a href="getfile?taskID=$task.id&name=$urllib.quote($filename)&offset=-4000">tail</a>)
#end if