From c6e69b4f8ba66b5ddb9e76b3d01ad4e5bd3aa141 Mon Sep 17 00:00:00 2001 From: Tomas Kopecek Date: Fri, 12 Feb 2021 15:34:26 +0100 Subject: [PATCH] backport py27 compatible file open with encoding client and builder needs to run on py27 which doesn't support open(encoding='utf-8') Related: https://pagure.io/koji/issue/2641 --- builder/kojid | 52 +++++++++++----------- builder/mergerepos | 8 ++-- cli/koji_cli/commands.py | 8 ++-- koji/__init__.py | 19 +++++--- koji/arch.py | 2 +- koji/daemon.py | 6 +-- plugins/builder/runroot.py | 4 +- tests/test_cli/test_mock_config.py | 2 +- tests/test_plugins/test_runroot_builder.py | 12 +---- vm/kojikamid.py | 8 ++-- 10 files changed, 62 insertions(+), 59 deletions(-) diff --git a/builder/kojid b/builder/kojid index efa293d3..bad674d4 100755 --- a/builder/kojid +++ b/builder/kojid @@ -310,7 +310,7 @@ class BuildRoot(object): output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts) # write config - with open(configfile, 'wt', encoding='utf-8') as fo: + with koji._open_text_file(configfile, 'wt') as fo: fo.write(output) def _repositoryEntries(self, pi, plugin=False): @@ -409,8 +409,8 @@ class BuildRoot(object): """ settings = settings % locals() - with open(self.rootdir() + destfile, 'wt') as fo: - fo.write(settings, encoding='utf-8') + with koji._open_text_file(self.rootdir() + destfile, 'wt') as fo: + fo.write(settings) def mock(self, args): """Run mock""" @@ -456,13 +456,13 @@ class BuildRoot(object): ts_name = '%s-ts.log' % fname fpath = os.path.join(resultdir, ts_name) if os.path.exists(fpath): - with open(fpath, 'rt', encoding='utf-8') as ts_file: + with koji._open_text_file(fpath) as ts_file: lines = ts_file.readlines() if lines: last = int(lines[-1].split()[1]) ts_offsets[fname] = last else: - with open(fpath, 'at', encoding='utf-8') as ts_file: + with koji._open_text_file(fpath, 'at') as ts_file: ts_file.write('%.0f 0\n' % time.time()) logs[ts_name] = (None, None, 0, fpath) if workdir and mocklog not in logs: @@ -473,13 +473,13 @@ class BuildRoot(object): ts_name = '%s-ts.log' % mocklog fpath = os.path.join(workdir, ts_name) if os.path.exists(fpath): - with open(fpath, 'rt', encoding='utf-8') as ts_file: + with koji._open_text_file(fpath) as ts_file: lines = ts_file.readlines() if lines: last = int(lines[-1].split()[1]) ts_offsets[mocklog] = last else: - with open(fpath, 'at', encoding='utf-8') as ts_file: + with koji._open_text_file(fpath, 'at') as ts_file: ts_file.write('%.0f 0\n' % time.time()) logs[ts_name] = (None, None, 0, fpath) @@ -510,7 +510,7 @@ class BuildRoot(object): ts_offsets.setdefault(fname, 0) if ts_offsets[fname] < position: fpath = os.path.join(resultdir, '%s-ts.log' % fname) - with open(fpath, 'at', encoding='utf-8') as ts_file: + with koji._open_text_file(fpath, 'at') as ts_file: ts_file.write('%.0f %i\n' % (time.time(), position)) ts_offsets[fname] = position incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger) @@ -1830,7 +1830,7 @@ class WrapperRPMTask(BaseBuildTask): tgt[field] = src.get(field) def spec_sanity_checks(self, filename): - spec = open(filename, encoding='utf-8').read() + spec = koji._open_text_file(filename).read() for tag in ("Packager", "Distribution", "Vendor"): if re.match("%s:" % tag, spec, re.M): raise koji.BuildError("%s is not allowed to be set in spec file" % tag) @@ -3055,7 +3055,7 @@ class ImageTask(BaseTaskHandler): kskoji = os.path.join(broot.tmpdir(), 'koji-image-%s-%i.ks' % (target_info['build_tag_name'], self.id)) koji.ensuredir(broot.tmpdir()) - with open(kskoji, 'wt', encoding='utf-8') as outfile: + with koji._open_text_file(kskoji, 'wt') as outfile: outfile.write(str(self.ks.handler)) # put the new ksfile in the output directory @@ -3253,7 +3253,7 @@ class LiveCDTask(ImageTask): Using iso9660 from pycdio, get the file manifest of the given image, and save it to the text file manifile. """ - fd = open(manifile, 'wt', encoding='utf-8') + fd = koji._open_text_file(manifile, 'wt') if not fd: raise koji.GenericError( 'Unable to open manifest file (%s) for writing!' % manifile) @@ -3442,7 +3442,7 @@ class LiveMediaTask(ImageTask): Using iso9660 from pycdio, get the file manifest of the given image, and save it to the text file manifile. """ - fd = open(manifile, 'wt', encoding='utf-8') + fd = koji._open_text_file(manifile, 'wt') if not fd: raise koji.GenericError( 'Unable to open manifest file (%s) for writing!' % manifile) @@ -3775,7 +3775,7 @@ class OzImageTask(BaseTaskHandler): an absolute path to the kickstart file we wrote """ kspath = os.path.join(self.workdir, ksname) - with open(kspath, 'wt', encoding='utf-8') as outfile: + with koji._open_text_file(kspath, 'wt') as outfile: outfile.write(str(ksobj.handler)) # put the new ksfile in the output directory @@ -3909,7 +3909,7 @@ class OzImageTask(BaseTaskHandler): edriver = newxml.getElementsByTagName('driver')[0] edriver.setAttribute('type', format) xml_path = os.path.join(self.workdir, filename) - with open(xml_path, 'wt', encoding='utf-8') as xmlfd: + with koji._open_text_file(xml_path, 'wt') as xmlfd: xmlfd.write(newxml.toprettyxml()) return xml_path @@ -4359,7 +4359,7 @@ class BaseImageTask(OzImageTask): ApplicationConfiguration(configuration=config) tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % self.arch) - with open(tdl_path, 'wt', encoding='utf-8') as tdl: + with koji._open_text_file(tdl_path, 'wt') as tdl: tdl.write(template) self.uploadFile(tdl_path) @@ -4563,7 +4563,7 @@ class BuildIndirectionImageTask(OzImageTask): # Factory doesn't attempt to modify a disk image after it is COMPLETE so # this will work safely on read-only NFS mounts factory_base_image.data = diskimage_full - factory_base_image.template = open(tdl_full, encoding='utf-8').read() + factory_base_image.template = koji._open_text_file(tdl_full).read() factory_base_image.status = 'COMPLETE' # Now save it pim.save_image(factory_base_image) @@ -4615,7 +4615,7 @@ class BuildIndirectionImageTask(OzImageTask): # Factory doesn't attempt to modify a disk image after it is COMPLETE so # this will work safely on read-only NFS mounts factory_base_image.data = diskimage_full - factory_base_image.template = open(tdl_full, encoding='utf-8').read() + factory_base_image.template = koji._open_text_file(tdl_full).read() factory_base_image.status = 'COMPLETE' # Now save it pim.save_image(factory_base_image) @@ -4705,7 +4705,7 @@ class BuildIndirectionImageTask(OzImageTask): rm = ReservationManager() rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT) - utility_customizations = open(indirection_template, encoding='utf-8').read() + utility_customizations = koji._open_text_file(indirection_template).read() results_loc = opts.get('results_loc', None) if results_loc[0] != "/": results_loc = "/" + results_loc @@ -4723,7 +4723,7 @@ class BuildIndirectionImageTask(OzImageTask): pim = PersistentImageManager.default_manager() pim.add_image(target_image) target.target_image = target_image - with open(target_image.data, "wt", encoding='utf-8') as f: + with koji._open_text_file(target_image.data, "wt") as f: f.write("Mock build from task ID: %s" % self.id) target_image.status = 'COMPLETE' else: @@ -4883,7 +4883,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask): _taskWeight = 1.0 def spec_sanity_checks(self, filename): - spec = open(filename, encoding='utf-8').read() + spec = koji._open_text_file(filename).read() for tag in ("Packager", "Distribution", "Vendor"): if re.match("%s:" % tag, spec, re.M): raise koji.BuildError("%s is not allowed to be set in spec file" % tag) @@ -5928,7 +5928,7 @@ enabled=1 # step 3: proceed with dnf config and set up yconfig_path = os.path.join(dnfdir, 'dnf.conf-koji-%s' % arch) - with open(yconfig_path, 'wt', encoding='utf-8') as f: + with koji._open_text_file(yconfig_path, 'wt') as f: f.write(dnfconfig) self.session.uploadWrapper(yconfig_path, self.uploadpath, os.path.basename(yconfig_path)) @@ -5965,7 +5965,7 @@ enabled=1 if len(fs_missing) > 0: missing_log = os.path.join(self.workdir, 'missing_multilib.log') - with open(missing_log, 'wt', encoding='utf-8') as outfile: + with koji._open_text_file(missing_log, 'wt') as outfile: outfile.write('The following multilib files were missing:\n') for ml_path in fs_missing: outfile.write(ml_path + '\n') @@ -6068,7 +6068,7 @@ enabled=1 # report problems if len(fs_missing) > 0: missing_log = os.path.join(self.workdir, 'missing_files.log') - with open(missing_log, 'wt', encoding='utf-8') as outfile: + with koji._open_text_file(missing_log, 'wt') as outfile: outfile.write('Some rpm files were missing.\n' 'Most likely, you want to create these signed copies.\n\n' 'Missing files:\n') @@ -6081,7 +6081,7 @@ enabled=1 if sig_missing: # log missing signatures and possibly error missing_log = os.path.join(self.workdir, 'missing_signatures.log') - with open(missing_log, 'wt', encoding='utf-8') as outfile: + with koji._open_text_file(missing_log, 'wt') as outfile: outfile.write('Some rpms were missing requested signatures.\n') if opts['skip_missing_signatures']: outfile.write('The skip_missing_signatures option was specified, so ' @@ -6130,12 +6130,12 @@ enabled=1 else: pkgs.append('Packages/%s/%s\n' % (bnplet, bnp)) - with open('%s/pkglist' % self.repodir, 'wt', encoding='utf-8') as fo: + with koji._open_text_file('%s/pkglist' % self.repodir, 'wt') as fo: for line in pkgs: fo.write(line) for subrepo in subrepo_pkgs: koji.ensuredir('%s/%s' % (self.repodir, subrepo)) - with open('%s/%s/pkglist' % (self.repodir, subrepo), 'wt', encoding='utf-8') as fo: + with koji._open_text_file('%s/%s/pkglist' % (self.repodir, subrepo), 'wt') as fo: for line in subrepo_pkgs[subrepo]: fo.write(line) diff --git a/builder/mergerepos b/builder/mergerepos index 539537ca..c6abc1e9 100755 --- a/builder/mergerepos +++ b/builder/mergerepos @@ -34,6 +34,8 @@ import rpmUtils.miscutils import yum import yum.misc +import koji + # Expand a canonical arch to the full list of # arches that should be included in the repo. # Basically the inverse of koji.canonArch(). @@ -252,7 +254,7 @@ class RepoMerge(object): include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid) pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins') - origins = open(pkgorigins, 'wt', encoding='utf-8') + origins = koji._open_text_file(pkgorigins, 'wt') seen_rpms = {} for repo in repos: @@ -307,7 +309,7 @@ class RepoMerge(object): pkg._return_remote_location = make_const_func(loc) pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins') - origins = open(pkgorigins, 'wt', encoding='utf-8') + origins = koji._open_text_file(pkgorigins, 'wt') seen_rpms = {} for repo in repos: @@ -348,7 +350,7 @@ def main(args): opts = parse_args(args) if opts.blocked: - with open(opts.blocked, encoding='utf-8') as blocked_fo: + with koji._open_text_file(opts.blocked) as blocked_fo: blocked_list = blocked_fo.readlines() blocked = dict([(b.strip(), 1) for b in blocked_list]) else: diff --git a/cli/koji_cli/commands.py b/cli/koji_cli/commands.py index 8b159795..71189905 100644 --- a/cli/koji_cli/commands.py +++ b/cli/koji_cli/commands.py @@ -1035,7 +1035,7 @@ def anon_handle_mock_config(goptions, session, args): opts['tag_macros'][macro] = buildcfg['extra'][key] output = koji.genMockConfig(name, arch, **opts) if options.ofile: - with open(options.ofile, 'wt', encoding='utf-8') as fo: + with open(options.ofile, 'wt') as fo: fo.write(output) else: print(output) @@ -1586,10 +1586,10 @@ def handle_prune_signed_copies(goptions, session, args): # (with the modification that we check to see if the build was latest within # the last N days) if options.ignore_tag_file: - with open(options.ignore_tag_file, encoding='utf-8') as fo: + with open(options.ignore_tag_file) as fo: options.ignore_tag.extend([line.strip() for line in fo.readlines()]) if options.protect_tag_file: - with open(options.protect_tag_file, encoding='utf-8') as fo: + with open(options.protect_tag_file) as fo: options.protect_tag.extend([line.strip() for line in fo.readlines()]) if options.debug: options.verbose = True @@ -6890,7 +6890,7 @@ def anon_handle_download_logs(options, session, args): full_filename = os.path.normpath(os.path.join(task_log_dir, FAIL_LOG)) koji.ensuredir(os.path.dirname(full_filename)) sys.stdout.write("Writing: %s\n" % full_filename) - with open(full_filename, 'wt', encoding='utf-8') as fo: + with open(full_filename, 'wt') as fo: fo.write(content) def download_log(task_log_dir, task_id, filename, blocksize=102400, volume=None): diff --git a/koji/__init__.py b/koji/__init__.py index fe5a8e2c..d2bbbe86 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -49,6 +49,7 @@ import warnings import weakref import xml.sax import xml.sax.handler +from contextlib import contextmanager from fnmatch import fnmatch import dateutil.parser @@ -1266,6 +1267,15 @@ class POMHandler(xml.sax.handler.ContentHandler): self.values.clear() +# BEGIN kojikamid dup # +def _open_text_file(path, mode='rt'): + # enforce utf-8 encoding for py3 + if six.PY2: + return open(path, mode) + else: + return open(path, mode, encoding='utf-8') +# END kojikamid dup $ + ENTITY_RE = re.compile(r'&[A-Za-z0-9]+;') @@ -1283,8 +1293,7 @@ def parse_pom(path=None, contents=None): values = {} handler = POMHandler(values, fields) if path: - with open(path, encoding='utf-8') as fd: - contents = fd.read() + contents = _open_text_file(path).read() if not contents: raise GenericError( @@ -1355,12 +1364,12 @@ def hex_string(s): def load_json(filepath): """Loads json from file""" - return json.load(open(filepath, 'rt', encoding='utf-8')) + return json.load(_open_text_file(filepath)) def dump_json(filepath, data, indent=4, sort_keys=False): """Write json to file""" - json.dump(data, open(filepath, 'wt', encoding='utf-8'), indent=indent, sort_keys=sort_keys) + json.dump(data, _open_text_file(filepath, 'wt'), indent=indent, sort_keys=sort_keys) def make_groups_spec(grplist, name='buildsys-build', buildgroup=None): @@ -1621,7 +1630,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts) if opts.get('use_host_resolv', False) and os.path.exists('/etc/hosts'): # if we're setting up DNS, # also copy /etc/hosts from the host - files['etc/hosts'] = open('/etc/hosts', 'rt', encoding='utf-8').read() + files['etc/hosts'] = _open_text_file('/etc/hosts').read() mavenrc = '' if opts.get('maven_opts'): mavenrc = 'export MAVEN_OPTS="%s"\n' % ' '.join(opts['maven_opts']) diff --git a/koji/arch.py b/koji/arch.py index 44daf424..65dd1c18 100644 --- a/koji/arch.py +++ b/koji/arch.py @@ -383,7 +383,7 @@ def getCanonX86_64Arch(arch): def getCanonArch(skipRpmPlatform=0): if not skipRpmPlatform and os.access("/etc/rpm/platform", os.R_OK): try: - with open("/etc/rpm/platform", "rt", encoding='utf-8') as f: + with open("/etc/rpm/platform", "rt") as f: line = f.readline() (arch, vendor, opersys) = line.split("-", 2) return arch diff --git a/koji/daemon.py b/koji/daemon.py index c4876532..bcd01b21 100644 --- a/koji/daemon.py +++ b/koji/daemon.py @@ -790,7 +790,7 @@ class TaskManager(object): fn = "%s/%s" % (configdir, f) if not os.path.isfile(fn): continue - fo = open(fn, 'rt', encoding='utf-8') + fo = koji._open_text_file(fn) id = None name = None for n in range(10): @@ -1089,7 +1089,7 @@ class TaskManager(object): proc_path = '/proc/%i/stat' % pid if not os.path.isfile(proc_path): return None - proc_file = open(proc_path, 'rt', encoding='utf-8') + proc_file = koji._open_text_file(proc_path) procstats = [not field.isdigit() and field or int(field) for field in proc_file.read().split()] proc_file.close() @@ -1097,7 +1097,7 @@ class TaskManager(object): cmd_path = '/proc/%i/cmdline' % pid if not os.path.isfile(cmd_path): return None - cmd_file = open(cmd_path, 'rt', encoding='utf-8') + cmd_file = koji._open_text_file(cmd_path) procstats[1] = cmd_file.read().replace('\0', ' ').strip() cmd_file.close() if not procstats[1]: diff --git a/plugins/builder/runroot.py b/plugins/builder/runroot.py index 7ff12123..0b098df2 100644 --- a/plugins/builder/runroot.py +++ b/plugins/builder/runroot.py @@ -267,7 +267,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): self.logger.info('New runroot') self.logger.info("Runroot mounts: %s" % mounts) fn = '%s/tmp/runroot_mounts' % rootdir - with open(fn, 'at', encoding='utf-8') as fslog: + with koji._open_text_file(fn, 'at') as fslog: logfile = "%s/do_mounts.log" % self.workdir uploadpath = self.getUploadDir() error = None @@ -309,7 +309,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler): mounts = set() fn = '%s/tmp/runroot_mounts' % rootdir if os.path.exists(fn): - with open(fn, 'rt', encoding='utf-8') as fslog: + with koji._open_text_file(fn) as fslog: for line in fslog.readlines(): mounts.add(line.strip()) # also, check /proc/mounts just in case diff --git a/tests/test_cli/test_mock_config.py b/tests/test_cli/test_mock_config.py index 753a95ef..67a527e7 100644 --- a/tests/test_cli/test_mock_config.py +++ b/tests/test_cli/test_mock_config.py @@ -333,7 +333,7 @@ config_opts['macros']['%distribution'] = 'Koji Testing' fobj = mock.MagicMock() openf.return_value.__enter__.return_value = fobj anon_handle_mock_config(options, session, arguments) - openf.assert_called_with('/tmp/mock.out', 'wt', encoding='utf-8') + openf.assert_called_once() fobj.write.assert_called_once_with(self.mock_output) gen_config_mock.assert_called_with( self.progname, arch, **opts) diff --git a/tests/test_plugins/test_runroot_builder.py b/tests/test_plugins/test_runroot_builder.py index 24998cbe..d87b37e2 100644 --- a/tests/test_plugins/test_runroot_builder.py +++ b/tests/test_plugins/test_runroot_builder.py @@ -13,14 +13,6 @@ __main__.BuildRoot = kojid.BuildRoot import koji import runroot -def mock_open(): - """Return the right patch decorator for open""" - if six.PY2: - return mock.patch('__builtin__.open') - else: - return mock.patch('builtins.open') - - if six.PY2: CONFIG_PARSER = 'six.moves.configparser.SafeConfigParser' else: @@ -192,7 +184,7 @@ class TestMounts(unittest.TestCase): self.assertEqual(self.t._get_path_params('/mnt/archive', 'rw'), ('archive.org:/vol/archive/', '/mnt/archive', 'nfs', 'rw,hard,intr,nosuid,nodev,noatime,tcp')) - @mock_open() + @mock.patch('koji._open_text_file') @mock.patch('os.path.isdir') @mock.patch('runroot.log_output') def test_do_mounts(self, log_output, is_dir, open_mock): @@ -293,7 +285,7 @@ class TestMounts(unittest.TestCase): self.t.do_mounts.assert_not_called() - @mock_open() + @mock.patch('koji._open_text_file') @mock.patch('runroot.scan_mounts') @mock.patch('os.unlink') @mock.patch('subprocess.Popen') diff --git a/vm/kojikamid.py b/vm/kojikamid.py index 46d54a44..caf604d0 100755 --- a/vm/kojikamid.py +++ b/vm/kojikamid.py @@ -331,7 +331,7 @@ class WindowsBuild(object): raise BuildError('Unknown checksum type %s for %s' % ( # noqa: F821 checksum_type, os.path.basename(fileinfo['localpath']))) - with open(destpath, 'wt', encoding='utf-8') as destfile: + with _open_text_file(destpath, 'wt') as destfile: offset = 0 while True: encoded = self.server.getFile(buildinfo, fileinfo, encode_int(offset), 1048576, @@ -412,7 +412,7 @@ class WindowsBuild(object): """Do the build: run the execute line(s) with cmd.exe""" tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp', suffix='.bat', dir='/cygdrive/c/Windows/Temp') - script = os.fdopen(tmpfd, 'wt', encoding='utf-8') + script = _open_text_file(tmpfd, 'wt') for attr in ['source_dir', 'spec_dir', 'patches_dir']: val = getattr(self, attr) if val: @@ -449,7 +449,7 @@ class WindowsBuild(object): def bashBuild(self): """Do the build: run the execute line(s) with bash""" tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp.', dir='/tmp') - script = os.fdopen(tmpfd, 'wt', encoding='utf-8') + script = _open_text_file(tmpfd, 'wt') script.write("export source_dir='%s'\n" % self.source_dir) script.write("export spec_dir='%s'\n" % self.spec_dir) if self.patches_dir: @@ -657,7 +657,7 @@ def setup_logging(opts): if opts.debug: level = logging.DEBUG logger.setLevel(level) - logfd = open(logfile, 'wt', encoding='utf-8') + logfd = _open_text_file(logfile, 'wt') handler = logging.StreamHandler(logfd) handler.setLevel(level) handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s'))