particle-os/debian-koji
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

PR#2693: backport py27 compatible file open with encoding

Browse source 
Merges #2693
https://pagure.io/koji/pull-request/2693

Fixes: #2641
https://pagure.io/koji/issue/2641
"open" encoding is defined outside of koji
This commit is contained in:
Tomas Kopecek 2021-02-16 14:13:09 +01:00
commit ce0c1b8b9b
10 changed files with 62 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

52
builder/kojid
View file

@ -310,7 +310,7 @@ class BuildRoot(object):
output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts)
# write config
with open(configfile, 'wt', encoding='utf-8') as fo:
with koji._open_text_file(configfile, 'wt') as fo:
fo.write(output)
def _repositoryEntries(self, pi, plugin=False):
@ -409,8 +409,8 @@ class BuildRoot(object):
</settings>
"""
settings = settings % locals()
with open(self.rootdir() + destfile, 'wt') as fo:
fo.write(settings, encoding='utf-8')
with koji._open_text_file(self.rootdir() + destfile, 'wt') as fo:
fo.write(settings)
def mock(self, args):
"""Run mock"""
@ -456,13 +456,13 @@ class BuildRoot(object):
ts_name = '%s-ts.log' % fname
fpath = os.path.join(resultdir, ts_name)
if os.path.exists(fpath):
with open(fpath, 'rt', encoding='utf-8') as ts_file:
with koji._open_text_file(fpath) as ts_file:
lines = ts_file.readlines()
if lines:
last = int(lines[-1].split()[1])
ts_offsets[fname] = last
else:
with open(fpath, 'at', encoding='utf-8') as ts_file:
with koji._open_text_file(fpath, 'at') as ts_file:
ts_file.write('%.0f 0\n' % time.time())
logs[ts_name] = (None, None, 0, fpath)
if workdir and mocklog not in logs:
@ -473,13 +473,13 @@ class BuildRoot(object):
ts_name = '%s-ts.log' % mocklog
fpath = os.path.join(workdir, ts_name)
if os.path.exists(fpath):
with open(fpath, 'rt', encoding='utf-8') as ts_file:
with koji._open_text_file(fpath) as ts_file:
lines = ts_file.readlines()
if lines:
last = int(lines[-1].split()[1])
ts_offsets[mocklog] = last
else:
with open(fpath, 'at', encoding='utf-8') as ts_file:
with koji._open_text_file(fpath, 'at') as ts_file:
ts_file.write('%.0f 0\n' % time.time())
logs[ts_name] = (None, None, 0, fpath)
@ -510,7 +510,7 @@ class BuildRoot(object):
ts_offsets.setdefault(fname, 0)
if ts_offsets[fname] < position:
fpath = os.path.join(resultdir, '%s-ts.log' % fname)
with open(fpath, 'at', encoding='utf-8') as ts_file:
with koji._open_text_file(fpath, 'at') as ts_file:
ts_file.write('%.0f %i\n' % (time.time(), position))
ts_offsets[fname] = position
incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger)
@ -1830,7 +1830,7 @@ class WrapperRPMTask(BaseBuildTask):
tgt[field] = src.get(field)
def spec_sanity_checks(self, filename):
spec = open(filename, encoding='utf-8').read()
spec = koji._open_text_file(filename).read()
for tag in ("Packager", "Distribution", "Vendor"):
if re.match("%s:" % tag, spec, re.M):
raise koji.BuildError("%s is not allowed to be set in spec file" % tag)
@ -3055,7 +3055,7 @@ class ImageTask(BaseTaskHandler):
kskoji = os.path.join(broot.tmpdir(), 'koji-image-%s-%i.ks' %
(target_info['build_tag_name'], self.id))
koji.ensuredir(broot.tmpdir())
with open(kskoji, 'wt', encoding='utf-8') as outfile:
with koji._open_text_file(kskoji, 'wt') as outfile:
outfile.write(str(self.ks.handler))
# put the new ksfile in the output directory
@ -3253,7 +3253,7 @@ class LiveCDTask(ImageTask):
Using iso9660 from pycdio, get the file manifest of the given image,
and save it to the text file manifile.
"""
fd = open(manifile, 'wt', encoding='utf-8')
fd = koji._open_text_file(manifile, 'wt')
if not fd:
raise koji.GenericError(
'Unable to open manifest file (%s) for writing!' % manifile)
@ -3442,7 +3442,7 @@ class LiveMediaTask(ImageTask):
Using iso9660 from pycdio, get the file manifest of the given image,
and save it to the text file manifile.
"""
fd = open(manifile, 'wt', encoding='utf-8')
fd = koji._open_text_file(manifile, 'wt')
if not fd:
raise koji.GenericError(
'Unable to open manifest file (%s) for writing!' % manifile)
@ -3775,7 +3775,7 @@ class OzImageTask(BaseTaskHandler):
an absolute path to the kickstart file we wrote
"""
kspath = os.path.join(self.workdir, ksname)
with open(kspath, 'wt', encoding='utf-8') as outfile:
with koji._open_text_file(kspath, 'wt') as outfile:
outfile.write(str(ksobj.handler))
# put the new ksfile in the output directory
@ -3909,7 +3909,7 @@ class OzImageTask(BaseTaskHandler):
edriver = newxml.getElementsByTagName('driver')[0]
edriver.setAttribute('type', format)
xml_path = os.path.join(self.workdir, filename)
with open(xml_path, 'wt', encoding='utf-8') as xmlfd:
with koji._open_text_file(xml_path, 'wt') as xmlfd:
xmlfd.write(newxml.toprettyxml())
return xml_path
@ -4359,7 +4359,7 @@ class BaseImageTask(OzImageTask):
ApplicationConfiguration(configuration=config)
tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % self.arch)
with open(tdl_path, 'wt', encoding='utf-8') as tdl:
with koji._open_text_file(tdl_path, 'wt') as tdl:
tdl.write(template)
self.uploadFile(tdl_path)
@ -4563,7 +4563,7 @@ class BuildIndirectionImageTask(OzImageTask):
# Factory doesn't attempt to modify a disk image after it is COMPLETE so
# this will work safely on read-only NFS mounts
factory_base_image.data = diskimage_full
factory_base_image.template = open(tdl_full, encoding='utf-8').read()
factory_base_image.template = koji._open_text_file(tdl_full).read()
factory_base_image.status = 'COMPLETE'
# Now save it
pim.save_image(factory_base_image)
@ -4615,7 +4615,7 @@ class BuildIndirectionImageTask(OzImageTask):
# Factory doesn't attempt to modify a disk image after it is COMPLETE so
# this will work safely on read-only NFS mounts
factory_base_image.data = diskimage_full
factory_base_image.template = open(tdl_full, encoding='utf-8').read()
factory_base_image.template = koji._open_text_file(tdl_full).read()
factory_base_image.status = 'COMPLETE'
# Now save it
pim.save_image(factory_base_image)
@ -4705,7 +4705,7 @@ class BuildIndirectionImageTask(OzImageTask):
rm = ReservationManager()
rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT)
utility_customizations = open(indirection_template, encoding='utf-8').read()
utility_customizations = koji._open_text_file(indirection_template).read()
results_loc = opts.get('results_loc', None)
if results_loc[0] != "/":
results_loc = "/" + results_loc
@ -4723,7 +4723,7 @@ class BuildIndirectionImageTask(OzImageTask):
pim = PersistentImageManager.default_manager()
pim.add_image(target_image)
target.target_image = target_image
with open(target_image.data, "wt", encoding='utf-8') as f:
with koji._open_text_file(target_image.data, "wt") as f:
f.write("Mock build from task ID: %s" % self.id)
target_image.status = 'COMPLETE'
else:
@ -4883,7 +4883,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask):
_taskWeight = 1.0
def spec_sanity_checks(self, filename):
spec = open(filename, encoding='utf-8').read()
spec = koji._open_text_file(filename).read()
for tag in ("Packager", "Distribution", "Vendor"):
if re.match("%s:" % tag, spec, re.M):
raise koji.BuildError("%s is not allowed to be set in spec file" % tag)
@ -5928,7 +5928,7 @@ enabled=1
# step 3: proceed with dnf config and set up
yconfig_path = os.path.join(dnfdir, 'dnf.conf-koji-%s' % arch)
with open(yconfig_path, 'wt', encoding='utf-8') as f:
with koji._open_text_file(yconfig_path, 'wt') as f:
f.write(dnfconfig)
self.session.uploadWrapper(yconfig_path, self.uploadpath,
os.path.basename(yconfig_path))
@ -5965,7 +5965,7 @@ enabled=1
if len(fs_missing) > 0:
missing_log = os.path.join(self.workdir, 'missing_multilib.log')
with open(missing_log, 'wt', encoding='utf-8') as outfile:
with koji._open_text_file(missing_log, 'wt') as outfile:
outfile.write('The following multilib files were missing:\n')
for ml_path in fs_missing:
outfile.write(ml_path + '\n')
@ -6068,7 +6068,7 @@ enabled=1
# report problems
if len(fs_missing) > 0:
missing_log = os.path.join(self.workdir, 'missing_files.log')
with open(missing_log, 'wt', encoding='utf-8') as outfile:
with koji._open_text_file(missing_log, 'wt') as outfile:
outfile.write('Some rpm files were missing.\n'
'Most likely, you want to create these signed copies.\n\n'
'Missing files:\n')
@ -6081,7 +6081,7 @@ enabled=1
if sig_missing:
# log missing signatures and possibly error
missing_log = os.path.join(self.workdir, 'missing_signatures.log')
with open(missing_log, 'wt', encoding='utf-8') as outfile:
with koji._open_text_file(missing_log, 'wt') as outfile:
outfile.write('Some rpms were missing requested signatures.\n')
if opts['skip_missing_signatures']:
outfile.write('The skip_missing_signatures option was specified, so '
@ -6130,12 +6130,12 @@ enabled=1
else:
pkgs.append('Packages/%s/%s\n' % (bnplet, bnp))
with open('%s/pkglist' % self.repodir, 'wt', encoding='utf-8') as fo:
with koji._open_text_file('%s/pkglist' % self.repodir, 'wt') as fo:
for line in pkgs:
fo.write(line)
for subrepo in subrepo_pkgs:
koji.ensuredir('%s/%s' % (self.repodir, subrepo))
with open('%s/%s/pkglist' % (self.repodir, subrepo), 'wt', encoding='utf-8') as fo:
with koji._open_text_file('%s/%s/pkglist' % (self.repodir, subrepo), 'wt') as fo:
for line in subrepo_pkgs[subrepo]:
fo.write(line)

8
builder/mergerepos
View file

@ -34,6 +34,8 @@ import rpmUtils.miscutils
import yum
import yum.misc
import koji
# Expand a canonical arch to the full list of
# arches that should be included in the repo.
# Basically the inverse of koji.canonArch().
@ -252,7 +254,7 @@ class RepoMerge(object):
include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid)
pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins')
origins = open(pkgorigins, 'wt', encoding='utf-8')
origins = koji._open_text_file(pkgorigins, 'wt')
seen_rpms = {}
for repo in repos:
@ -307,7 +309,7 @@ class RepoMerge(object):
pkg._return_remote_location = make_const_func(loc)
pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins')
origins = open(pkgorigins, 'wt', encoding='utf-8')
origins = koji._open_text_file(pkgorigins, 'wt')
seen_rpms = {}
for repo in repos:
@ -348,7 +350,7 @@ def main(args):
opts = parse_args(args)
if opts.blocked:
with open(opts.blocked, encoding='utf-8') as blocked_fo:
with koji._open_text_file(opts.blocked) as blocked_fo:
blocked_list = blocked_fo.readlines()
blocked = dict([(b.strip(), 1) for b in blocked_list])
else:

8
cli/koji_cli/commands.py
View file

@ -1035,7 +1035,7 @@ def anon_handle_mock_config(goptions, session, args):
opts['tag_macros'][macro] = buildcfg['extra'][key]
output = koji.genMockConfig(name, arch, **opts)
if options.ofile:
with open(options.ofile, 'wt', encoding='utf-8') as fo:
with open(options.ofile, 'wt') as fo:
fo.write(output)
else:
print(output)
@ -1586,10 +1586,10 @@ def handle_prune_signed_copies(goptions, session, args):
# (with the modification that we check to see if the build was latest within
# the last N days)
if options.ignore_tag_file:
with open(options.ignore_tag_file, encoding='utf-8') as fo:
with open(options.ignore_tag_file) as fo:
options.ignore_tag.extend([line.strip() for line in fo.readlines()])
if options.protect_tag_file:
with open(options.protect_tag_file, encoding='utf-8') as fo:
with open(options.protect_tag_file) as fo:
options.protect_tag.extend([line.strip() for line in fo.readlines()])
if options.debug:
options.verbose = True
@ -6890,7 +6890,7 @@ def anon_handle_download_logs(options, session, args):
full_filename = os.path.normpath(os.path.join(task_log_dir, FAIL_LOG))
koji.ensuredir(os.path.dirname(full_filename))
sys.stdout.write("Writing: %s\n" % full_filename)
with open(full_filename, 'wt', encoding='utf-8') as fo:
with open(full_filename, 'wt') as fo:
fo.write(content)
def download_log(task_log_dir, task_id, filename, blocksize=102400, volume=None):

19
koji/__init__.py
View file

@ -49,6 +49,7 @@ import warnings
import weakref
import xml.sax
import xml.sax.handler
from contextlib import contextmanager
from fnmatch import fnmatch
import dateutil.parser
@ -1266,6 +1267,15 @@ class POMHandler(xml.sax.handler.ContentHandler):
self.values.clear()
# BEGIN kojikamid dup #
def _open_text_file(path, mode='rt'):
# enforce utf-8 encoding for py3
if six.PY2:
return open(path, mode)
else:
return open(path, mode, encoding='utf-8')
# END kojikamid dup $
ENTITY_RE = re.compile(r'&[A-Za-z0-9]+;')
@ -1283,8 +1293,7 @@ def parse_pom(path=None, contents=None):
values = {}
handler = POMHandler(values, fields)
if path:
with open(path, encoding='utf-8') as fd:
contents = fd.read()
contents = _open_text_file(path).read()
if not contents:
raise GenericError(
@ -1355,12 +1364,12 @@ def hex_string(s):
def load_json(filepath):
"""Loads json from file"""
return json.load(open(filepath, 'rt', encoding='utf-8'))
return json.load(_open_text_file(filepath))
def dump_json(filepath, data, indent=4, sort_keys=False):
"""Write json to file"""
json.dump(data, open(filepath, 'wt', encoding='utf-8'), indent=indent, sort_keys=sort_keys)
json.dump(data, _open_text_file(filepath, 'wt'), indent=indent, sort_keys=sort_keys)
def make_groups_spec(grplist, name='buildsys-build', buildgroup=None):
@ -1621,7 +1630,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts)
if opts.get('use_host_resolv', False) and os.path.exists('/etc/hosts'):
# if we're setting up DNS,
# also copy /etc/hosts from the host
files['etc/hosts'] = open('/etc/hosts', 'rt', encoding='utf-8').read()
files['etc/hosts'] = _open_text_file('/etc/hosts').read()
mavenrc = ''
if opts.get('maven_opts'):
mavenrc = 'export MAVEN_OPTS="%s"\n' % ' '.join(opts['maven_opts'])

2
koji/arch.py
View file

@ -383,7 +383,7 @@ def getCanonX86_64Arch(arch):
def getCanonArch(skipRpmPlatform=0):
if not skipRpmPlatform and os.access("/etc/rpm/platform", os.R_OK):
try:
with open("/etc/rpm/platform", "rt", encoding='utf-8') as f:
with open("/etc/rpm/platform", "rt") as f:
line = f.readline()
(arch, vendor, opersys) = line.split("-", 2)
return arch

6
koji/daemon.py
View file

@ -790,7 +790,7 @@ class TaskManager(object):
fn = "%s/%s" % (configdir, f)
if not os.path.isfile(fn):
continue
fo = open(fn, 'rt', encoding='utf-8')
fo = koji._open_text_file(fn)
id = None
name = None
for n in range(10):
@ -1089,7 +1089,7 @@ class TaskManager(object):
proc_path = '/proc/%i/stat' % pid
if not os.path.isfile(proc_path):
return None
proc_file = open(proc_path, 'rt', encoding='utf-8')
proc_file = koji._open_text_file(proc_path)
procstats = [not field.isdigit() and field or int(field)
for field in proc_file.read().split()]
proc_file.close()
@ -1097,7 +1097,7 @@ class TaskManager(object):
cmd_path = '/proc/%i/cmdline' % pid
if not os.path.isfile(cmd_path):
return None
cmd_file = open(cmd_path, 'rt', encoding='utf-8')
cmd_file = koji._open_text_file(cmd_path)
procstats[1] = cmd_file.read().replace('\0', ' ').strip()
cmd_file.close()
if not procstats[1]:

4
plugins/builder/runroot.py
View file

@ -267,7 +267,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler):
self.logger.info('New runroot')
self.logger.info("Runroot mounts: %s" % mounts)
fn = '%s/tmp/runroot_mounts' % rootdir
with open(fn, 'at', encoding='utf-8') as fslog:
with koji._open_text_file(fn, 'at') as fslog:
logfile = "%s/do_mounts.log" % self.workdir
uploadpath = self.getUploadDir()
error = None
@ -309,7 +309,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler):
mounts = set()
fn = '%s/tmp/runroot_mounts' % rootdir
if os.path.exists(fn):
with open(fn, 'rt', encoding='utf-8') as fslog:
with koji._open_text_file(fn) as fslog:
for line in fslog.readlines():
mounts.add(line.strip())
# also, check /proc/mounts just in case

2
tests/test_cli/test_mock_config.py
View file

@ -333,7 +333,7 @@ config_opts['macros']['%distribution'] = 'Koji Testing'
fobj = mock.MagicMock()
openf.return_value.__enter__.return_value = fobj
anon_handle_mock_config(options, session, arguments)
openf.assert_called_with('/tmp/mock.out', 'wt', encoding='utf-8')
openf.assert_called_once()
fobj.write.assert_called_once_with(self.mock_output)
gen_config_mock.assert_called_with(
self.progname, arch, **opts)

12
tests/test_plugins/test_runroot_builder.py
View file

@ -13,14 +13,6 @@ __main__.BuildRoot = kojid.BuildRoot
import koji
import runroot
def mock_open():
"""Return the right patch decorator for open"""
if six.PY2:
return mock.patch('__builtin__.open')
else:
return mock.patch('builtins.open')
if six.PY2:
CONFIG_PARSER = 'six.moves.configparser.SafeConfigParser'
else:
@ -192,7 +184,7 @@ class TestMounts(unittest.TestCase):
self.assertEqual(self.t._get_path_params('/mnt/archive', 'rw'),
('archive.org:/vol/archive/', '/mnt/archive', 'nfs', 'rw,hard,intr,nosuid,nodev,noatime,tcp'))
@mock_open()
@mock.patch('koji._open_text_file')
@mock.patch('os.path.isdir')
@mock.patch('runroot.log_output')
def test_do_mounts(self, log_output, is_dir, open_mock):
@ -293,7 +285,7 @@ class TestMounts(unittest.TestCase):
self.t.do_mounts.assert_not_called()
@mock_open()
@mock.patch('koji._open_text_file')
@mock.patch('runroot.scan_mounts')
@mock.patch('os.unlink')
@mock.patch('subprocess.Popen')

8
vm/kojikamid.py
View file

@ -331,7 +331,7 @@ class WindowsBuild(object):
raise BuildError('Unknown checksum type %s for %s' % ( # noqa: F821
checksum_type,
os.path.basename(fileinfo['localpath'])))
with open(destpath, 'wt', encoding='utf-8') as destfile:
with _open_text_file(destpath, 'wt') as destfile:
offset = 0
while True:
encoded = self.server.getFile(buildinfo, fileinfo, encode_int(offset), 1048576,
@ -412,7 +412,7 @@ class WindowsBuild(object):
"""Do the build: run the execute line(s) with cmd.exe"""
tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp', suffix='.bat',
dir='/cygdrive/c/Windows/Temp')
script = os.fdopen(tmpfd, 'wt', encoding='utf-8')
script = _open_text_file(tmpfd, 'wt')
for attr in ['source_dir', 'spec_dir', 'patches_dir']:
val = getattr(self, attr)
if val:
@ -449,7 +449,7 @@ class WindowsBuild(object):
def bashBuild(self):
"""Do the build: run the execute line(s) with bash"""
tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp.', dir='/tmp')
script = os.fdopen(tmpfd, 'wt', encoding='utf-8')
script = _open_text_file(tmpfd, 'wt')
script.write("export source_dir='%s'\n" % self.source_dir)
script.write("export spec_dir='%s'\n" % self.spec_dir)
if self.patches_dir:
@ -657,7 +657,7 @@ def setup_logging(opts):
if opts.debug:
level = logging.DEBUG
logger.setLevel(level)
logfd = open(logfile, 'wt', encoding='utf-8')
logfd = _open_text_file(logfile, 'wt')
handler = logging.StreamHandler(logfd)
handler.setLevel(level)
handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s'))