particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

skip fixEncoding for py3

Browse source
This commit is contained in:
Tomas Kopecek 2019-02-27 17:13:40 +01:00 • committed by Mike McLean
parent 613d1587a6
commit eeeeb3ab78
6 changed files with 76 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

36
hub/kojihub.py
View file

@ -4361,10 +4361,7 @@ def _get_zipfile_list(archive_id, zippath):
return result return result
with zipfile.ZipFile(zippath, 'r') as archive: with zipfile.ZipFile(zippath, 'r') as archive:
for entry in archive.infolist(): for entry in archive.infolist():
if six.PY2: filename = koji.fixEncoding(entry.filename)
filename = koji.fixEncoding(entry.filename)
else:
filename = entry.filename
result.append({'archive_id': archive_id, result.append({'archive_id': archive_id,
'name': filename, 'name': filename,
'size': entry.file_size, 'size': entry.file_size,
@ -4389,10 +4386,7 @@ def _get_tarball_list(archive_id, tarpath):
return result return result
with tarfile.open(tarpath, 'r') as archive: with tarfile.open(tarpath, 'r') as archive:
for entry in archive: for entry in archive:
if six.PY2: filename = koji.fixEncoding(entry.name)
filename = koji.fixEncoding(entry.name)
else:
filename = entry.name
result.append({'archive_id': archive_id, result.append({'archive_id': archive_id,
'name': filename, 'name': filename,
'size': entry.size, 'size': entry.size,
@ -6368,10 +6362,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No
archiveinfo = {'buildroot_id': buildroot_id} archiveinfo = {'buildroot_id': buildroot_id}
archiveinfo['build_id'] = buildinfo['id'] archiveinfo['build_id'] = buildinfo['id']
if metadata_only: if metadata_only:
if six.PY2: filename = koji.fixEncoding(fileinfo['filename'])
filename = koji.fixEncoding(fileinfo['filename'])
else:
filename = fileinfo['filename']
archiveinfo['filename'] = filename archiveinfo['filename'] = filename
archiveinfo['size'] = fileinfo['filesize'] archiveinfo['size'] = fileinfo['filesize']
archiveinfo['checksum'] = fileinfo['checksum'] archiveinfo['checksum'] = fileinfo['checksum']
@ -6382,10 +6373,7 @@ def import_archive_internal(filepath, buildinfo, type, typeInfo, buildroot_id=No
archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES[fileinfo['checksum_type']] archiveinfo['checksum_type'] = koji.CHECKSUM_TYPES[fileinfo['checksum_type']]
archiveinfo['metadata_only'] = True archiveinfo['metadata_only'] = True
else: else:
if six.PY2: filename = koji.fixEncoding(os.path.basename(filepath))
filename = koji.fixEncoding(os.path.basename(filepath))
else:
filename = os.path.basename(filepath)
archiveinfo['filename'] = filename archiveinfo['filename'] = filename
archiveinfo['size'] = os.path.getsize(filepath) archiveinfo['size'] = os.path.getsize(filepath)
# trust values computed on hub (CG_Importer.prep_outputs) # trust values computed on hub (CG_Importer.prep_outputs)
@ -6507,8 +6495,7 @@ def _import_archive_file(filepath, destdir):
be created. be created.
""" """
fname = os.path.basename(filepath) fname = os.path.basename(filepath)
if six.PY2: fname = koji.fixEncoding(fname)
fname = koji.fixEncoding(fname)
final_path = "%s/%s" % (destdir, fname) final_path = "%s/%s" % (destdir, fname)
if os.path.exists(final_path): if os.path.exists(final_path):
raise koji.GenericError("Error importing archive file, %s already exists" % final_path) raise koji.GenericError("Error importing archive file, %s already exists" % final_path)
@ -7693,10 +7680,7 @@ def parse_json(value, desc=None, errstr=None):
if value is None: if value is None:
return value return value
try: try:
if six.PY2: return koji.fixEncodingRecurse(json.loads(value))
return koji.fixEncodingRecurse(json.loads(value))
else:
return json.loads(value)
except Exception: except Exception:
if errstr is None: if errstr is None:
if desc is None: if desc is None:
@ -9698,9 +9682,8 @@ class RootExports(object):
for (cltime, clname, cltext) in zip(fields['changelogtime'], fields['changelogname'], for (cltime, clname, cltext) in zip(fields['changelogtime'], fields['changelogname'],
fields['changelogtext']): fields['changelogtext']):
cldate = datetime.datetime.fromtimestamp(cltime).isoformat(' ') cldate = datetime.datetime.fromtimestamp(cltime).isoformat(' ')
if six.PY2: clname = koji.fixEncoding(clname)
clname = koji.fixEncoding(clname) cltext = koji.fixEncoding(cltext)
cltext = koji.fixEncoding(cltext)
if author and author != clname: if author and author != clname:
continue continue
@ -10260,8 +10243,7 @@ class RootExports(object):
raise koji.GenericError('either rpmID or taskID and filepath must be specified') raise koji.GenericError('either rpmID or taskID and filepath must be specified')
headers = koji.get_header_fields(rpm_path, headers) headers = koji.get_header_fields(rpm_path, headers)
return koji.fixEncodingRecurse(headers, remove_nonprintable=True, return koji.fixEncodingRecurse(headers, remove_nonprintable=True)
ignore_keys=True)
queryRPMSigs = staticmethod(query_rpm_sigs) queryRPMSigs = staticmethod(query_rpm_sigs)

26
koji/__init__.py
View file

@ -3063,10 +3063,16 @@ def _taskLabel(taskInfo):
return '%s (%s)' % (method, arch) return '%s (%s)' % (method, arch)
CONTROL_CHARS = [chr(i) for i in range(32)] CONTROL_CHARS = [chr(i) for i in range(32)]
NONPRINTABLE_CHARS = six.b(''.join([c for c in CONTROL_CHARS if c not in '\r\n\t'])) NONPRINTABLE_CHARS = ''.join([c for c in CONTROL_CHARS if c not in '\r\n\t'])
if six.PY3:
NONPRINTABLE_CHARS_TABLE = dict.fromkeys(map(ord, NONPRINTABLE_CHARS), None)
def removeNonprintable(value): def removeNonprintable(value):
# expects raw-encoded string, not unicode # expects raw-encoded string, not unicode
return value.translate(None, NONPRINTABLE_CHARS) if six.PY2:
return value.translate(None, NONPRINTABLE_CHARS)
else:
return value.translate(NONPRINTABLE_CHARS_TABLE)
def _fix_print(value): def _fix_print(value):
@ -3089,6 +3095,12 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False):
If value is not valid UTF-8 to begin with, assume it is If value is not valid UTF-8 to begin with, assume it is
encoded in the 'fallback' charset. encoded in the 'fallback' charset.
""" """
if six.PY3:
if remove_nonprintable:
return removeNonprintable(value)
else:
return value
if not value: if not value:
return six.b('') return six.b('')
@ -3110,11 +3122,15 @@ def fixEncoding(value, fallback='iso8859-15', remove_nonprintable=False):
return s return s
def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False, ignore_keys=False): def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False):
"""Recursively fix string encoding in an object """Recursively fix string encoding in an object
Similar behavior to fixEncoding, but recursive Similar behavior to fixEncoding, but recursive
""" """
if six.PY3:
# don't bother with fixing in py3
return value
if isinstance(value, tuple): if isinstance(value, tuple):
return tuple([fixEncodingRecurse(x, fallback=fallback, remove_nonprintable=remove_nonprintable) for x in value]) return tuple([fixEncodingRecurse(x, fallback=fallback, remove_nonprintable=remove_nonprintable) for x in value])
elif isinstance(value, list): elif isinstance(value, list):
@ -3123,9 +3139,7 @@ def fixEncodingRecurse(value, fallback='iso8859-15', remove_nonprintable=False,
ret = {} ret = {}
for k in value: for k in value:
v = fixEncodingRecurse(value[k], fallback=fallback, remove_nonprintable=remove_nonprintable) v = fixEncodingRecurse(value[k], fallback=fallback, remove_nonprintable=remove_nonprintable)
# expect, that keys are never really binary k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable)
if not ignore_keys:
k = fixEncodingRecurse(k, fallback=fallback, remove_nonprintable=remove_nonprintable)
ret[k] = v ret[k] = v
return ret return ret
elif isinstance(value, six.text_type): elif isinstance(value, six.text_type):

2
tests/test_builder/test_build_notification.py
View file

@ -100,5 +100,5 @@ class TestBuildNotification(unittest.TestCase):
self.assertEqual(recipients, ["user@example.com"]) self.assertEqual(recipients, ["user@example.com"])
fn = os.path.join(os.path.dirname(__file__), 'data/calls', 'build_notif_1', 'message.txt') fn = os.path.join(os.path.dirname(__file__), 'data/calls', 'build_notif_1', 'message.txt')
with open(fn, 'rb') as fp: with open(fn, 'rb') as fp:
msg_expect = fp.read() msg_expect = fp.read().decode()
self.assertEqual(message, msg_expect) self.assertEqual(message, msg_expect)

87
tests/test_lib/test_fixEncoding.py
View file

@ -18,52 +18,48 @@ class FixEncodingTestCase(unittest.TestCase):
simple_values = [ simple_values = [
# [ value, fixed ] # [ value, fixed ]
['', six.b('')], ['', ''],
[u'', six.b('')], [u'', ''],
[u'góðan daginn', six.b('g\xc3\xb3\xc3\xb0an daginn')], [u'góðan daginn', 'g\xc3\xb3\xc3\xb0an daginn'],
[u'hej', six.b('hej')], [u'hej', 'hej'],
[u'zdravstvuite', six.b('zdravstvuite')], [u'zdravstvuite', 'zdravstvuite'],
[u'céad míle fáilte', six.b('c\xc3\xa9ad m\xc3\xadle f\xc3\xa1ilte')], [u'céad míle fáilte', 'c\xc3\xa9ad m\xc3\xadle f\xc3\xa1ilte'],
[u'dobrý den', six.b('dobr\xc3\xbd den')], [u'dobrý den', 'dobr\xc3\xbd den'],
[u'hylô', six.b('hyl\xc3\xb4')], [u'hylô', 'hyl\xc3\xb4'],
[u'jó napot', six.b('j\xc3\xb3 napot')], [u'jó napot', 'j\xc3\xb3 napot'],
[u'tervehdys', six.b('tervehdys')], [u'tervehdys', 'tervehdys'],
[u'olá', six.b('ol\xc3\xa1')], [u'olá', 'ol\xc3\xa1'],
[u'grüezi', six.b('gr\xc3\xbcezi')], [u'grüezi', 'gr\xc3\xbcezi'],
[u'dobre dan', six.b('dobre dan')], [u'dobre dan', 'dobre dan'],
[u'hello', six.b('hello')], [u'hello', 'hello'],
[u'bună ziua', six.b('bun\xc4\x83 ziua')], [u'bună ziua', 'bun\xc4\x83 ziua'],
[u'こんにちは', six.b('\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf')], [u'こんにちは', '\xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf'],
[u'你好', six.b('\xe4\xbd\xa0\xe5\xa5\xbd')], [u'你好', '\xe4\xbd\xa0\xe5\xa5\xbd'],
[u'नमस्कार', six.b('\xe0\xa4\xa8\xe0\xa4\xae\xe0\xa4\xb8\xe0\xa5\x8d\xe0\xa4\x95\xe0\xa4\xbe\xe0\xa4\xb0')], [u'नमस्कार', '\xe0\xa4\xa8\xe0\xa4\xae\xe0\xa4\xb8\xe0\xa5\x8d\xe0\xa4\x95\xe0\xa4\xbe\xe0\xa4\xb0'],
[u'안녕하세요', six.b('\xec\x95\x88\xeb\x85\x95\xed\x95\x98\xec\x84\xb8\xec\x9a\x94')], [u'안녕하세요', '\xec\x95\x88\xeb\x85\x95\xed\x95\x98\xec\x84\xb8\xec\x9a\x94'],
] ]
def test_fixEncoding(self): def test_fixEncoding(self):
"""Test the fixEncoding function""" """Test the fixEncoding function"""
for a, b in self.simple_values: for a, b in self.simple_values:
self.assertEqual(koji.fixEncoding(a), b)
self.assertEqual(koji.fixEncoding(b), b) self.assertEqual(koji.fixEncoding(b), b)
c = a.encode('utf16') if six.PY2:
self.assertEqual(koji.fixEncoding(c, fallback='utf16'), b) self.assertEqual(koji.fixEncoding(a), b)
d = a[:-3] + u'\x00\x01' + a[-3:] c = a.encode('utf16')
self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b) self.assertEqual(koji.fixEncoding(c, fallback='utf16'), b)
d = a[:-3] + u'\x00\x01' + a[-3:]
self.assertEqual(koji.fixEncoding(d, remove_nonprintable=True), b)
else:
self.assertEqual(koji.fixEncoding(a), a)
@mock.patch('sys.stdout', new_callable=six.StringIO) def test_fix_print(self):
def test_fix_print(self, stdout):
"""Test the _fix_print function""" """Test the _fix_print function"""
expected = '' actual, expected = [], []
for a, b in self.simple_values: for a, b in self.simple_values:
if six.PY3: actual.append(koji._fix_print(b))
self.assertEqual(koji._fix_print(b), a) expected.append(b)
else: expected = '\n'.join(expected)
self.assertEqual(koji._fix_print(b), b) actual = '\n'.join(actual)
print(koji._fix_print(b))
if six.PY3:
expected = expected + a + '\n'
else:
expected = expected + b + '\n'
actual = stdout.getvalue()
self.assertEqual(actual, expected) self.assertEqual(actual, expected)
complex_values = [ complex_values = [
@ -73,21 +69,24 @@ class FixEncodingTestCase(unittest.TestCase):
[None, None], [None, None],
[[], []], [[], []],
[{u'a': 'a' , 'b' : {'c': u'c\x00'}}, [{u'a': 'a' , 'b' : {'c': u'c\x00'}},
{six.b('a'): six.b('a') , six.b('b') : {six.b('c'): six.b('c\x00')}}], {'a': 'a' , 'b' : {'c': 'c\x00'}}],
# iso8859-15 fallback # iso8859-15 fallback
['g\xf3\xf0an daginn', six.b('g\xc3\xb3\xc3\xb0an daginn')], ['g\xf3\xf0an daginn', 'g\xc3\xb3\xc3\xb0an daginn'],
] ]
nonprint = [ nonprint = [
['hello\0world\0', six.b('helloworld')], ['hello\0world\0', 'helloworld'],
[u'hello\0world\0', six.b('helloworld')], [u'hello\0world\0', 'helloworld'],
[[u'hello\0world\0'], [six.b('helloworld')]], [[u'hello\0world\0'], ['helloworld']],
[{0: u'hello\0world\0'}, {0: six.b('helloworld')}], [{0: u'hello\0world\0'}, {0: 'helloworld'}],
[[{0: u'hello\0world\0'}], [{0: six.b('helloworld')}]], [[{0: u'hello\0world\0'}], [{0: 'helloworld'}]],
] ]
def test_fixEncodingRecurse(self): def test_fixEncodingRecurse(self):
"""Test the fixEncodingRecurse function""" """Test the fixEncodingRecurse function"""
if six.PY3:
# don't test for py3
return
for a, b in self.simple_values: for a, b in self.simple_values:
self.assertEqual(koji.fixEncoding(a), b) self.assertEqual(koji.fixEncoding(a), b)
for a, b in self.complex_values: for a, b in self.complex_values:

6
www/kojiweb/index.py
View file

@ -1382,9 +1382,9 @@ def rpminfo(environ, rpmID, fileOrder='name', fileStart=None, buildrootOrder='-i
values['enhances'] = server.getRPMDeps(rpm['id'], koji.DEP_ENHANCE) values['enhances'] = server.getRPMDeps(rpm['id'], koji.DEP_ENHANCE)
values['enhances'].sort(key=_sortbyname) values['enhances'].sort(key=_sortbyname)
headers = server.getRPMHeaders(rpm['id'], headers=['summary', 'description', 'license']) headers = server.getRPMHeaders(rpm['id'], headers=['summary', 'description', 'license'])
values['summary'] = koji.fixEncoding(str(headers.get('summary'))) values['summary'] = koji.fixEncoding(headers.get('summary'))
values['description'] = koji.fixEncoding(str(headers.get('description'))) values['description'] = koji.fixEncoding(headers.get('description'))
values['license'] = koji.fixEncoding(str(headers.get('license'))) values['license'] = koji.fixEncoding(headers.get('license'))
buildroots = kojiweb.util.paginateMethod(server, values, 'listBuildroots', kw={'rpmID': rpm['id']}, buildroots = kojiweb.util.paginateMethod(server, values, 'listBuildroots', kw={'rpmID': rpm['id']},
start=buildrootStart, dataName='buildroots', prefix='buildroot', start=buildrootStart, dataName='buildroots', prefix='buildroot',
order=buildrootOrder) order=buildrootOrder)

3
www/lib/kojiweb/util.py
View file

@ -531,9 +531,6 @@ def escapeHTML(value):
return value return value
value = koji.fixEncoding(value) value = koji.fixEncoding(value)
if six.PY3:
# it is bytes now, so decode to str
value = value.decode()
return value.replace('&', '&').\ return value.replace('&', '&').\
replace('<', '&lt;').\ replace('<', '&lt;').\
replace('>', '&gt;') replace('>', '&gt;')