particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix downloads w/o content-length

Browse source 
Fixes: https://pagure.io/koji/issue/982
This commit is contained in:
Tomas Kopecek 2018-06-25 10:53:01 +02:00
parent ad61fdee12
commit 98b83eff69
3 changed files with 14 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

19
cli/koji_cli/lib.py
View file

@ -487,26 +487,20 @@ def download_file(url, relpath, quiet=False, noprogress=False, size=None, num=No
else:
print(_("Downloading: %s") % relpath)
# closing needs to be used for requests < 2.18.0
with closing(requests.get(url, stream=True)) as response:
# raise error if occured
response.raise_for_status()
length = response.headers.get('content-length')
f = open(relpath, 'wb')
if length is None:
f.write(response.content)
length = len(response.content)
if not (quiet or noprogress):
_download_progress(length, length)
else:
length = int(response.headers.get('content-length') or 0)
with open(relpath, 'wb') as f:
l = 0
length = int(length)
for chunk in response.iter_content(chunk_size=65536):
l += len(chunk)
f.write(chunk)
if not (quiet or noprogress):
_download_progress(length, l)
f.close()
if not length and not (quiet or noprogress):
_download_progress(l, l)
if not (quiet or noprogress):
print('')
@ -515,9 +509,10 @@ def download_file(url, relpath, quiet=False, noprogress=False, size=None, num=No
def _download_progress(download_t, download_d):
if download_t == 0:
percent_done = 0.0
percent_done_str = "???%"
else:
percent_done = float(download_d) / float(download_t)
percent_done_str = "%3d%%" % (percent_done * 100)
percent_done_str = "%3d%%" % (percent_done * 100)
data_done = _format_size(download_d)
sys.stdout.write("[% -36s] % 4s % 10s\r" % ('=' * (int(percent_done * 36)), percent_done_str, data_done))

8
tests/test_cli/test_download_file.py
View file

@ -84,18 +84,18 @@ class TestDownloadFile(unittest.TestCase):
response = mock.MagicMock()
self.requests_get.return_value = response
response.headers.get.return_value = None # content-length
response.content = 'abcdef'
response.iter_content.return_value = ['a' * 65536, 'b' * 65536]
rv = download_file("http://url", self.filename)
actual = self.stdout.getvalue()
expected = 'Downloading: %s\n[====================================] 100%% 6.00 B\r\n' % self.filename
expected = 'Downloading: %s\n[ ] ???%% 64.00 KiB\r[ ] ???%% 128.00 KiB\r[====================================] 100%% 128.00 KiB\r\n' % self.filename
self.assertMultiLineEqual(actual, expected)
self.requests_get.assert_called_once()
m_open.assert_called_once()
response.headers.get.assert_called_once()
response.iter_content.assert_not_called()
response.iter_content.assert_called_once()
self.assertIsNone(rv)
@ -148,7 +148,7 @@ class TestDownloadProgress(unittest.TestCase):
_download_progress(1024 * 1024 * 1024 * 35, 1024 * 1024 * 1024 * 30)
_download_progress(318921, 318921)
actual = self.stdout.getvalue()
expected = '[ ] 0% 0.00 B\r' + \
expected = '[ ] ???% 0.00 B\r' + \
'[ ] 1% 1.00 KiB\r' + \
'[================= ] 47% 11.00 MiB\r' + \
'[============================== ] 85% 30.00 GiB\r' + \

9
vm/kojivmd
View file

@ -680,14 +680,11 @@ class VMExecTask(BaseTaskHandler):
else:
raise koji.BuildError('unsupported file type: %s' % type)
koji.ensuredir(os.path.dirname(localpath))
# closing needs to be used for requests < 2.18.0
with closing(requests.get(remote_url, stream=True)) as response:
with open(localpath, 'wb') as f:
length = response.headers.get('content-length')
if length is None:
f.write(response.content)
else:
for chunk in response.iter_content(chunk_size=65536):
f.write(chunk)
for chunk in response.iter_content(chunk_size=65536):
f.write(chunk)
if type == 'rpm':
# rpm, check sigmd5. It is enough, as if content is broken,
# rpm will fail later