particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

skip fixEncoding for py3

Browse source
This commit is contained in:
Tomas Kopecek 2019-02-27 17:13:40 +01:00 • committed by Mike McLean
parent 613d1587a6
commit eeeeb3ab78
6 changed files with 76 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

36
hub/kojihub.py
View file

@ -4361,10 +4361,7 @@ def _get_zipfile_list(archive_id, zippath):
return result
with zipfile.ZipFile(zippath, 'r') as archive:
for entry in archive.infolist():
if six.PY2:
filename = koji.fixEncoding(entry.filename)
else:
filename = entry.filename
filename = koji.fixEncoding(entry.filename)
result.append({'archive_id': archive_id,
'name': filename,
'size': entry.file_size,
@ -4389,10 +4386,7 @@ def _get_tarball_list(archive_id, tarpath):
return result
with tarfile.open(tarpath, 'r') as archive:
for entry in archive:
if six.PY2:
filename = koji.fixEncoding(entry.name)
else:
filename = entry.name
filename = koji.fixEncoding(entry.name)
result.append({'archive_id': archive_id,
'name': filename,
'size': entry.size,
@ -6368,10 +6362,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No
archiveinfo = {'buildroot_id': buildroot_id}
archiveinfo['build_id'] = buildinfo['id']
if metadata_only:
if six.PY2:
filename = koji.fixEncoding(fileinfo['filename'])
else:
filename = fileinfo['filename']
filename = koji.fixEncoding(fileinfo['filename'])
archiveinfo['filename'] = filename
archiveinfo['size'] = fileinfo['filesize']
archiveinfo['checksum'] = fileinfo['checksum']
@ -6382,10 +6373,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No
archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES[fileinfo['checksum_type']]
archiveinfo['metadata_only'] = True
else:
if six.PY2:
filename = koji.fixEncoding(os.path.basename(filepath))
else:
filename = os.path.basename(filepath)
filename = koji.fixEncoding(os.path.basename(filepath))
archiveinfo['filename'] = filename
archiveinfo['size'] = os.path.getsize(filepath)
# trust values computed on hub (CG_Importer.prep_outputs)
@ -6507,8 +6495,7 @@ def _import_archive_file(filepath, destdir):
be created.
"""
fname = os.path.basename(filepath)
if six.PY2:
fname = koji.fixEncoding(fname)
fname = koji.fixEncoding(fname)
final_path = "%s/%s" % (destdir, fname)
if os.path.exists(final_path):
raise koji.GenericError("Error importing archive file, %s already exists" % final_path)
@ -7693,10 +7680,7 @@ def parse_json(value, desc=None, errstr=None):
if value is None:
return value
try:
if six.PY2:
return koji.fixEncodingRecurse(json.loads(value))
else:
return json.loads(value)
return koji.fixEncodingRecurse(json.loads(value))
except Exception:
if errstr is None:
if desc is None:
@ -9698,9 +9682,8 @@ class RootExports(object):
for (cltime, clname, cltext) in zip(fields['changelogtime'], fields['changelogname'],
fields['changelogtext']):
cldate = datetime.datetime.fromtimestamp(cltime).isoformat(' ')
if six.PY2:
clname = koji.fixEncoding(clname)
cltext = koji.fixEncoding(cltext)
clname = koji.fixEncoding(clname)
cltext = koji.fixEncoding(cltext)
if author and author != clname:
continue
@ -10260,8 +10243,7 @@ class RootExports(object):
raise koji.GenericError('either rpmID or taskID and filepath must be specified')
headers = koji.get_header_fields(rpm_path, headers)
return koji.fixEncodingRecurse(headers, remove_nonprintable=True,
ignore_keys=True)
return koji.fixEncodingRecurse(headers, remove_nonprintable=True)
queryRPMSigs = staticmethod(query_rpm_sigs)

26
koji/__init__.py
View file

@ -3063,10 +3063,16 @@ def _taskLabel(taskInfo):
return '%s (%s)' % (method, arch)
CONTROL_CHARS = [chr(i) for i in range(32)]
NONPRINTABLE_CHARS = six.b(''.join([c for c in CONTROL_CHARS if c not in '\r\n\t']))
NONPRINTABLE_CHARS = ''.join([c for c in CONTROL_CHARS if c not in '\r\n\t'])
if six.PY3:
NONPRINTABLE_CHARS_TABLE = dict.fromkeys(map(ord, NONPRINTABLE_CHARS), None)
def removeNonprintable(value):
# expects raw-encoded string, not unicode
return value.translate(None, NONPRINTABLE_CHARS)
if six.PY2:
return value.translate(None, NONPRINTABLE_CHARS)
else:
return value.translate(NONPRINTABLE_CHARS_TABLE)
def _fix_print(value):
@ -3089,6 +3095,12 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False):
If value is not valid UTF-8 to begin with, assume it is
encoded in the 'fallback' charset.
"""
if six.PY3:
if remove_nonprintable:
return removeNonprintable(value)
else:
return value
if not value:
return six.b('')
@ -3110,11 +3122,15 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False):
return s
def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False, ignore_keys=False):
def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False):
"""Recursively fix string encoding in an object
Similar behavior to fixEncoding, but recursive
"""
if six.PY3:
# don't bother with fixing in py3
return value
if isinstance(value, tuple):
return tuple([fixEncodingRecurse(x, fallback=fallback, remove_nonprintable=remove_nonprintable) for x in value])
elif isinstance(value, list):
@ -3123,9 +3139,7 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False,
ret = {}
for k in value:
v = fixEncodingRecurse(value[k], fallback=fallback, remove_nonprintable=remove_nonprintable)
# expect, that keys are never really binary
if not ignore_keys:
k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable)
k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable)
ret[k] = v
return ret
elif isinstance(value, six.text_type):

2
tests/test_builder/test_build_notification.py
View file

@ -100,5 +100,5 @@ class TestBuildNotification(unittest.TestCase):
self.assertEqual(recipients, ["user@example.com"])
fn = os.path.join(os.path.dirname(__file__), 'data/calls', 'build_notif_1', 'message.txt')
with open(fn, 'rb') as fp:
msg_expect = fp.read()
msg_expect = fp.read().decode()
self.assertEqual(message, msg_expect)

87
tests/test_lib/test_fixEncoding.py
View file

@ -18,52 +18,48 @@ class FixEncodingTestCase(unittest.TestCase):
simple_values = [
# [ value, fixed ]
['', six.b('')],
[u'', six.b('')],
[u'góðan daginn', six.b('g\xc3\xb3\xc3\xb0an daginn')],
[u'hej', six.b('hej')],
[u'zdravstvuite', six.b('zdravstvuite')],
[u'céad míle fáilte', six.b('c\xc3\xa9ad m\xc3\xadle f\xc3\xa1ilte')],
[u'dobrý den', six.b('dobr\xc3\xbd den')],
[u'hylô', six.b('hyl\xc3\xb4')],
[u'jó napot', six.b('j\xc3\xb3 napot')],
[u'tervehdys', six.b('tervehdys')],
[u'olá', six.b('ol\xc3\xa1')],
[u'grüezi', six.b('gr\xc3\xbcezi')],
[u'dobre dan', six.b('dobre dan')],
[u'hello', six.b('hello')],
[u'bună ziua', six.b('bun\xc4\x83 ziua')],
[u'こんにちは', six.b('\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf')],
[u'你好', six.b('\xe4\xbd\xa0\xe5\xa5\xbd')],
[u'नमस्कार', six.b('\xe0\xa4\xa8\xe0\xa4\xae\xe0\xa4\xb8\xe0\xa5\x8d\xe0\xa4\x95\xe0\xa4\xbe\xe0\xa4\xb0')],
[u'안녕하세요', six.b('\xec\x95\x88\xeb\x85\x95\xed\x95\x98\xec\x84\xb8\xec\x9a\x94')],
['', ''],
[u'', ''],
[u'góðan daginn', 'g\xc3\xb3\xc3\xb0an daginn'],
[u'hej', 'hej'],
[u'zdravstvuite', 'zdravstvuite'],
[u'céad míle fáilte', 'c\xc3\xa9ad m\xc3\xadle f\xc3\xa1ilte'],
[u'dobrý den', 'dobr\xc3\xbd den'],
[u'hylô', 'hyl\xc3\xb4'],
[u'jó napot', 'j\xc3\xb3 napot'],
[u'tervehdys', 'tervehdys'],
[u'olá', 'ol\xc3\xa1'],
[u'grüezi', 'gr\xc3\xbcezi'],
[u'dobre dan', 'dobre dan'],
[u'hello', 'hello'],
[u'bună ziua', 'bun\xc4\x83 ziua'],
[u'こんにちは', '\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf'],
[u'你好', '\xe4\xbd\xa0\xe5\xa5\xbd'],
[u'नमस्कार', '\xe0\xa4\xa8\xe0\xa4\xae\xe0\xa4\xb8\xe0\xa5\x8d\xe0\xa4\x95\xe0\xa4\xbe\xe0\xa4\xb0'],
[u'안녕하세요', '\xec\x95\x88\xeb\x85\x95\xed\x95\x98\xec\x84\xb8\xec\x9a\x94'],
]
def test_fixEncoding(self):
"""Test the fixEncoding function"""
for a, b in self.simple_values:
self.assertEqual(koji.fixEncoding(a), b)
self.assertEqual(koji.fixEncoding(b), b)
c = a.encode('utf16')
self.assertEqual(koji.fixEncoding(c, fallback='utf16'), b)
d = a[:-3] + u'\x00\x01' + a[-3:]
self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b)
if six.PY2:
self.assertEqual(koji.fixEncoding(a), b)
c = a.encode('utf16')
self.assertEqual(koji.fixEncoding(c, fallback='utf16'), b)
d = a[:-3] + u'\x00\x01' + a[-3:]
self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b)
else:
self.assertEqual(koji.fixEncoding(a), a)
@mock.patch('sys.stdout', new_callable=six.StringIO)
def test_fix_print(self, stdout):
def test_fix_print(self):
"""Test the _fix_print function"""
expected = ''
actual, expected = [], []
for a, b in self.simple_values:
if six.PY3:
self.assertEqual(koji._fix_print(b), a)
else:
self.assertEqual(koji._fix_print(b), b)
print(koji._fix_print(b))
if six.PY3:
expected = expected + a + '\n'
else:
expected = expected + b + '\n'
actual = stdout.getvalue()
actual.append(koji._fix_print(b))
expected.append(b)
expected = '\n'.join(expected)
actual = '\n'.join(actual)
self.assertEqual(actual, expected)
complex_values = [
@ -73,21 +69,24 @@ class FixEncodingTestCase(unittest.TestCase):
[None, None],
[[], []],
[{u'a': 'a' , 'b' : {'c': u'c\x00'}},
{six.b('a'): six.b('a') , six.b('b') : {six.b('c'): six.b('c\x00')}}],
{'a': 'a' , 'b' : {'c': 'c\x00'}}],
# iso8859-15 fallback
['g\xf3\xf0an daginn', six.b('g\xc3\xb3\xc3\xb0an daginn')],
['g\xf3\xf0an daginn', 'g\xc3\xb3\xc3\xb0an daginn'],
]
nonprint = [
['hello\0world\0', six.b('helloworld')],
[u'hello\0world\0', six.b('helloworld')],
[[u'hello\0world\0'], [six.b('helloworld')]],
[{0: u'hello\0world\0'}, {0: six.b('helloworld')}],
[[{0: u'hello\0world\0'}], [{0: six.b('helloworld')}]],
['hello\0world\0', 'helloworld'],
[u'hello\0world\0', 'helloworld'],
[[u'hello\0world\0'], ['helloworld']],
[{0: u'hello\0world\0'}, {0: 'helloworld'}],
[[{0: u'hello\0world\0'}], [{0: 'helloworld'}]],
]
def test_fixEncodingRecurse(self):
"""Test the fixEncodingRecurse function"""
if six.PY3:
# don't test for py3
return
for a, b in self.simple_values:
self.assertEqual(koji.fixEncoding(a), b)
for a, b in self.complex_values:

6
www/kojiweb/index.py
View file

@ -1382,9 +1382,9 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i
values['enhances'] = server.getRPMDeps(rpm['id'], koji.DEP_ENHANCE)
values['enhances'].sort(key=_sortbyname)
headers = server.getRPMHeaders(rpm['id'], headers=['summary', 'description', 'license'])
values['summary'] = koji.fixEncoding(str(headers.get('summary')))
values['description'] = koji.fixEncoding(str(headers.get('description')))
values['license'] = koji.fixEncoding(str(headers.get('license')))
values['summary'] = koji.fixEncoding(headers.get('summary'))
values['description'] = koji.fixEncoding(headers.get('description'))
values['license'] = koji.fixEncoding(headers.get('license'))
buildroots = kojiweb.util.paginateMethod(server, values, 'listBuildroots', kw={'rpmID': rpm['id']},
start=buildrootStart, dataName='buildroots', prefix='buildroot',
order=buildrootOrder)

3
www/lib/kojiweb/util.py
View file

@ -531,9 +531,6 @@ def escapeHTML(value):
return value
value = koji.fixEncoding(value)
if six.PY3:
# it is bytes now, so decode to str
value = value.decode()
return value.replace('&', '&').\
replace('<', '&lt;').\
replace('>', '&gt;')