backport py27 compatible file open with encoding

client and builder needs to run on py27 which doesn't support
open(encoding='utf-8')

Related: https://pagure.io/koji/issue/2641

builder/kojid

@@ -310,7 +310,7 @@ class BuildRoot(object):
         output = koji.genMockConfig(self.name, self.br_arch, managed=True, **opts)
 
         # write config
-        with open(configfile, 'wt', encoding='utf-8') as fo:
+        with koji._open_text_file(configfile, 'wt') as fo:
             fo.write(output)
 
     def _repositoryEntries(self, pi, plugin=False):
@@ -409,8 +409,8 @@ class BuildRoot(object):
 </settings>
 """
         settings = settings % locals()
-        with open(self.rootdir() + destfile, 'wt') as fo:
+        with koji._open_text_file(self.rootdir() + destfile, 'wt') as fo:
-            fo.write(settings, encoding='utf-8')
+            fo.write(settings)
 
     def mock(self, args):
         """Run mock"""
@@ -456,13 +456,13 @@ class BuildRoot(object):
                             ts_name = '%s-ts.log' % fname
                             fpath = os.path.join(resultdir, ts_name)
                             if os.path.exists(fpath):
-                                with open(fpath, 'rt', encoding='utf-8') as ts_file:
+                                with koji._open_text_file(fpath) as ts_file:
                                     lines = ts_file.readlines()
                                     if lines:
                                         last = int(lines[-1].split()[1])
                                         ts_offsets[fname] = last
                             else:
-                                with open(fpath, 'at', encoding='utf-8') as ts_file:
+                                with koji._open_text_file(fpath, 'at') as ts_file:
                                     ts_file.write('%.0f 0\n' % time.time())
                             logs[ts_name] = (None, None, 0, fpath)
                 if workdir and mocklog not in logs:
@@ -473,13 +473,13 @@ class BuildRoot(object):
                             ts_name = '%s-ts.log' % mocklog
                             fpath = os.path.join(workdir, ts_name)
                             if os.path.exists(fpath):
-                                with open(fpath, 'rt', encoding='utf-8') as ts_file:
+                                with koji._open_text_file(fpath) as ts_file:
                                     lines = ts_file.readlines()
                                     if lines:
                                         last = int(lines[-1].split()[1])
                                         ts_offsets[mocklog] = last
                             else:
-                                with open(fpath, 'at', encoding='utf-8') as ts_file:
+                                with koji._open_text_file(fpath, 'at') as ts_file:
                                     ts_file.write('%.0f 0\n' % time.time())
                             logs[ts_name] = (None, None, 0, fpath)
 
@@ -510,7 +510,7 @@ class BuildRoot(object):
                         ts_offsets.setdefault(fname, 0)
                         if ts_offsets[fname] < position:
                             fpath = os.path.join(resultdir, '%s-ts.log' % fname)
-                            with open(fpath, 'at', encoding='utf-8') as ts_file:
+                            with koji._open_text_file(fpath, 'at') as ts_file:
                                 ts_file.write('%.0f %i\n' % (time.time(), position))
                             ts_offsets[fname] = position
                     incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger)
@@ -1830,7 +1830,7 @@ class WrapperRPMTask(BaseBuildTask):
             tgt[field] = src.get(field)
 
     def spec_sanity_checks(self, filename):
-        spec = open(filename, encoding='utf-8').read()
+        spec = koji._open_text_file(filename).read()
         for tag in ("Packager", "Distribution", "Vendor"):
             if re.match("%s:" % tag, spec, re.M):
                 raise koji.BuildError("%s is not allowed to be set in spec file" % tag)
@@ -3055,7 +3055,7 @@ class ImageTask(BaseTaskHandler):
         kskoji = os.path.join(broot.tmpdir(), 'koji-image-%s-%i.ks' %
                               (target_info['build_tag_name'], self.id))
         koji.ensuredir(broot.tmpdir())
-        with open(kskoji, 'wt', encoding='utf-8') as outfile:
+        with koji._open_text_file(kskoji, 'wt') as outfile:
             outfile.write(str(self.ks.handler))
 
         # put the new ksfile in the output directory
@@ -3253,7 +3253,7 @@ class LiveCDTask(ImageTask):
         Using iso9660 from pycdio, get the file manifest of the given image,
         and save it to the text file manifile.
         """
-        fd = open(manifile, 'wt', encoding='utf-8')
+        fd = koji._open_text_file(manifile, 'wt')
         if not fd:
             raise koji.GenericError(
                 'Unable to open manifest file (%s) for writing!' % manifile)
@@ -3442,7 +3442,7 @@ class LiveMediaTask(ImageTask):
         Using iso9660 from pycdio, get the file manifest of the given image,
         and save it to the text file manifile.
         """
-        fd = open(manifile, 'wt', encoding='utf-8')
+        fd = koji._open_text_file(manifile, 'wt')
         if not fd:
             raise koji.GenericError(
                 'Unable to open manifest file (%s) for writing!' % manifile)
@@ -3775,7 +3775,7 @@ class OzImageTask(BaseTaskHandler):
             an absolute path to the kickstart file we wrote
         """
         kspath = os.path.join(self.workdir, ksname)
-        with open(kspath, 'wt', encoding='utf-8') as outfile:
+        with koji._open_text_file(kspath, 'wt') as outfile:
             outfile.write(str(ksobj.handler))
 
         # put the new ksfile in the output directory
@@ -3909,7 +3909,7 @@ class OzImageTask(BaseTaskHandler):
         edriver = newxml.getElementsByTagName('driver')[0]
         edriver.setAttribute('type', format)
         xml_path = os.path.join(self.workdir, filename)
-        with open(xml_path, 'wt', encoding='utf-8') as xmlfd:
+        with koji._open_text_file(xml_path, 'wt') as xmlfd:
             xmlfd.write(newxml.toprettyxml())
         return xml_path
 
@@ -4359,7 +4359,7 @@ class BaseImageTask(OzImageTask):
         ApplicationConfiguration(configuration=config)
 
         tdl_path = os.path.join(self.workdir, 'tdl-%s.xml' % self.arch)
-        with open(tdl_path, 'wt', encoding='utf-8') as tdl:
+        with koji._open_text_file(tdl_path, 'wt') as tdl:
             tdl.write(template)
         self.uploadFile(tdl_path)
 
@@ -4563,7 +4563,7 @@ class BuildIndirectionImageTask(OzImageTask):
             # Factory doesn't attempt to modify a disk image after it is COMPLETE so
             # this will work safely on read-only NFS mounts
             factory_base_image.data = diskimage_full
-            factory_base_image.template = open(tdl_full, encoding='utf-8').read()
+            factory_base_image.template = koji._open_text_file(tdl_full).read()
             factory_base_image.status = 'COMPLETE'
             # Now save it
             pim.save_image(factory_base_image)
@@ -4615,7 +4615,7 @@ class BuildIndirectionImageTask(OzImageTask):
             # Factory doesn't attempt to modify a disk image after it is COMPLETE so
             # this will work safely on read-only NFS mounts
             factory_base_image.data = diskimage_full
-            factory_base_image.template = open(tdl_full, encoding='utf-8').read()
+            factory_base_image.template = koji._open_text_file(tdl_full).read()
             factory_base_image.status = 'COMPLETE'
             # Now save it
             pim.save_image(factory_base_image)
@@ -4705,7 +4705,7 @@ class BuildIndirectionImageTask(OzImageTask):
         rm = ReservationManager()
         rm._listen_port = rm.MIN_PORT + self.id % (rm.MAX_PORT - rm.MIN_PORT)
 
-        utility_customizations = open(indirection_template, encoding='utf-8').read()
+        utility_customizations = koji._open_text_file(indirection_template).read()
         results_loc = opts.get('results_loc', None)
         if results_loc[0] != "/":
             results_loc = "/" + results_loc
@@ -4723,7 +4723,7 @@ class BuildIndirectionImageTask(OzImageTask):
                     pim = PersistentImageManager.default_manager()
                     pim.add_image(target_image)
                     target.target_image = target_image
-                    with open(target_image.data, "wt", encoding='utf-8') as f:
+                    with koji._open_text_file(target_image.data, "wt") as f:
                         f.write("Mock build from task ID: %s" % self.id)
                     target_image.status = 'COMPLETE'
                 else:
@@ -4883,7 +4883,7 @@ class BuildSRPMFromSCMTask(BaseBuildTask):
     _taskWeight = 1.0
 
     def spec_sanity_checks(self, filename):
-        spec = open(filename, encoding='utf-8').read()
+        spec = koji._open_text_file(filename).read()
         for tag in ("Packager", "Distribution", "Vendor"):
             if re.match("%s:" % tag, spec, re.M):
                 raise koji.BuildError("%s is not allowed to be set in spec file" % tag)
@@ -5928,7 +5928,7 @@ enabled=1
 
         # step 3: proceed with dnf config and set up
         yconfig_path = os.path.join(dnfdir, 'dnf.conf-koji-%s' % arch)
-        with open(yconfig_path, 'wt', encoding='utf-8') as f:
+        with koji._open_text_file(yconfig_path, 'wt') as f:
             f.write(dnfconfig)
         self.session.uploadWrapper(yconfig_path, self.uploadpath,
                                    os.path.basename(yconfig_path))
@@ -5965,7 +5965,7 @@ enabled=1
 
         if len(fs_missing) > 0:
             missing_log = os.path.join(self.workdir, 'missing_multilib.log')
-            with open(missing_log, 'wt', encoding='utf-8') as outfile:
+            with koji._open_text_file(missing_log, 'wt') as outfile:
                 outfile.write('The following multilib files were missing:\n')
                 for ml_path in fs_missing:
                     outfile.write(ml_path + '\n')
@@ -6068,7 +6068,7 @@ enabled=1
         # report problems
         if len(fs_missing) > 0:
             missing_log = os.path.join(self.workdir, 'missing_files.log')
-            with open(missing_log, 'wt', encoding='utf-8') as outfile:
+            with koji._open_text_file(missing_log, 'wt') as outfile:
                 outfile.write('Some rpm files were missing.\n'
                               'Most likely, you want to create these signed copies.\n\n'
                               'Missing files:\n')
@@ -6081,7 +6081,7 @@ enabled=1
         if sig_missing:
             # log missing signatures and possibly error
             missing_log = os.path.join(self.workdir, 'missing_signatures.log')
-            with open(missing_log, 'wt', encoding='utf-8') as outfile:
+            with koji._open_text_file(missing_log, 'wt') as outfile:
                 outfile.write('Some rpms were missing requested signatures.\n')
                 if opts['skip_missing_signatures']:
                     outfile.write('The skip_missing_signatures option was specified, so '
@@ -6130,12 +6130,12 @@ enabled=1
             else:
                 pkgs.append('Packages/%s/%s\n' % (bnplet, bnp))
 
-        with open('%s/pkglist' % self.repodir, 'wt', encoding='utf-8') as fo:
+        with koji._open_text_file('%s/pkglist' % self.repodir, 'wt') as fo:
             for line in pkgs:
                 fo.write(line)
         for subrepo in subrepo_pkgs:
             koji.ensuredir('%s/%s' % (self.repodir, subrepo))
-            with open('%s/%s/pkglist' % (self.repodir, subrepo), 'wt', encoding='utf-8') as fo:
+            with koji._open_text_file('%s/%s/pkglist' % (self.repodir, subrepo), 'wt') as fo:
                 for line in subrepo_pkgs[subrepo]:
                     fo.write(line)
 

builder/mergerepos

@@ -34,6 +34,8 @@ import rpmUtils.miscutils
 import yum
 import yum.misc
 
+import koji
+
 # Expand a canonical arch to the full list of
 # arches that should be included in the repo.
 # Basically the inverse of koji.canonArch().
@@ -252,7 +254,7 @@ class RepoMerge(object):
                     include_srpms[srpm_name] = (pkg.sourcerpm, pkg.repoid)
 
         pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins')
-        origins = open(pkgorigins, 'wt', encoding='utf-8')
+        origins = koji._open_text_file(pkgorigins, 'wt')
 
         seen_rpms = {}
         for repo in repos:
@@ -307,7 +309,7 @@ class RepoMerge(object):
                     pkg._return_remote_location = make_const_func(loc)
 
         pkgorigins = os.path.join(self.yumbase.conf.cachedir, 'pkgorigins')
-        origins = open(pkgorigins, 'wt', encoding='utf-8')
+        origins = koji._open_text_file(pkgorigins, 'wt')
 
         seen_rpms = {}
         for repo in repos:
@@ -348,7 +350,7 @@ def main(args):
     opts = parse_args(args)
 
     if opts.blocked:
-        with open(opts.blocked, encoding='utf-8') as blocked_fo:
+        with koji._open_text_file(opts.blocked) as blocked_fo:
             blocked_list = blocked_fo.readlines()
         blocked = dict([(b.strip(), 1) for b in blocked_list])
     else:

cli/koji_cli/commands.py

@@ -1035,7 +1035,7 @@ def anon_handle_mock_config(goptions, session, args):
             opts['tag_macros'][macro] = buildcfg['extra'][key]
     output = koji.genMockConfig(name, arch, **opts)
     if options.ofile:
-        with open(options.ofile, 'wt', encoding='utf-8') as fo:
+        with open(options.ofile, 'wt') as fo:
             fo.write(output)
     else:
         print(output)
@@ -1586,10 +1586,10 @@ def handle_prune_signed_copies(goptions, session, args):
     # (with the modification that we check to see if the build was latest within
     # the last N days)
     if options.ignore_tag_file:
-        with open(options.ignore_tag_file, encoding='utf-8') as fo:
+        with open(options.ignore_tag_file) as fo:
             options.ignore_tag.extend([line.strip() for line in fo.readlines()])
     if options.protect_tag_file:
-        with open(options.protect_tag_file, encoding='utf-8') as fo:
+        with open(options.protect_tag_file) as fo:
             options.protect_tag.extend([line.strip() for line in fo.readlines()])
     if options.debug:
         options.verbose = True
@@ -6890,7 +6890,7 @@ def anon_handle_download_logs(options, session, args):
         full_filename = os.path.normpath(os.path.join(task_log_dir, FAIL_LOG))
         koji.ensuredir(os.path.dirname(full_filename))
         sys.stdout.write("Writing: %s\n" % full_filename)
-        with open(full_filename, 'wt', encoding='utf-8') as fo:
+        with open(full_filename, 'wt') as fo:
             fo.write(content)
 
     def download_log(task_log_dir, task_id, filename, blocksize=102400, volume=None):

koji/__init__.py

@@ -49,6 +49,7 @@ import warnings
 import weakref
 import xml.sax
 import xml.sax.handler
+from contextlib import contextmanager
 from fnmatch import fnmatch
 
 import dateutil.parser
@@ -1266,6 +1267,15 @@ class POMHandler(xml.sax.handler.ContentHandler):
         self.values.clear()
 
 
+# BEGIN kojikamid dup #
+def _open_text_file(path, mode='rt'):
+    # enforce utf-8 encoding for py3
+    if six.PY2:
+        return open(path, mode)
+    else:
+        return open(path, mode, encoding='utf-8')
+# END kojikamid dup $
+
 ENTITY_RE = re.compile(r'&[A-Za-z0-9]+;')
 
 
@@ -1283,8 +1293,7 @@ def parse_pom(path=None, contents=None):
     values = {}
     handler = POMHandler(values, fields)
     if path:
-        with open(path, encoding='utf-8') as fd:
+        contents = _open_text_file(path).read()
-            contents = fd.read()
 
     if not contents:
         raise GenericError(
@@ -1355,12 +1364,12 @@ def hex_string(s):
 
 def load_json(filepath):
     """Loads json from file"""
-    return json.load(open(filepath, 'rt', encoding='utf-8'))
+    return json.load(_open_text_file(filepath))
 
 
 def dump_json(filepath, data, indent=4, sort_keys=False):
     """Write json to file"""
-    json.dump(data, open(filepath, 'wt', encoding='utf-8'), indent=indent, sort_keys=sort_keys)
+    json.dump(data, _open_text_file(filepath, 'wt'), indent=indent, sort_keys=sort_keys)
 
 
 def make_groups_spec(grplist, name='buildsys-build', buildgroup=None):
@@ -1621,7 +1630,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts)
     if opts.get('use_host_resolv', False) and os.path.exists('/etc/hosts'):
         # if we're setting up DNS,
         # also copy /etc/hosts from the host
-        files['etc/hosts'] = open('/etc/hosts', 'rt', encoding='utf-8').read()
+        files['etc/hosts'] = _open_text_file('/etc/hosts').read()
     mavenrc = ''
     if opts.get('maven_opts'):
         mavenrc = 'export MAVEN_OPTS="%s"\n' % ' '.join(opts['maven_opts'])

koji/arch.py

@@ -383,7 +383,7 @@ def getCanonX86_64Arch(arch):
 def getCanonArch(skipRpmPlatform=0):
     if not skipRpmPlatform and os.access("/etc/rpm/platform", os.R_OK):
         try:
-            with open("/etc/rpm/platform", "rt", encoding='utf-8') as f:
+            with open("/etc/rpm/platform", "rt") as f:
                 line = f.readline()
             (arch, vendor, opersys) = line.split("-", 2)
             return arch

koji/daemon.py

@@ -790,7 +790,7 @@ class TaskManager(object):
             fn = "%s/%s" % (configdir, f)
             if not os.path.isfile(fn):
                 continue
-            fo = open(fn, 'rt', encoding='utf-8')
+            fo = koji._open_text_file(fn)
             id = None
             name = None
             for n in range(10):
@@ -1089,7 +1089,7 @@ class TaskManager(object):
             proc_path = '/proc/%i/stat' % pid
             if not os.path.isfile(proc_path):
                 return None
-            proc_file = open(proc_path, 'rt', encoding='utf-8')
+            proc_file = koji._open_text_file(proc_path)
             procstats = [not field.isdigit() and field or int(field)
                          for field in proc_file.read().split()]
             proc_file.close()
@@ -1097,7 +1097,7 @@ class TaskManager(object):
             cmd_path = '/proc/%i/cmdline' % pid
             if not os.path.isfile(cmd_path):
                 return None
-            cmd_file = open(cmd_path, 'rt', encoding='utf-8')
+            cmd_file = koji._open_text_file(cmd_path)
             procstats[1] = cmd_file.read().replace('\0', ' ').strip()
             cmd_file.close()
             if not procstats[1]:

plugins/builder/runroot.py

@@ -267,7 +267,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler):
         self.logger.info('New runroot')
         self.logger.info("Runroot mounts: %s" % mounts)
         fn = '%s/tmp/runroot_mounts' % rootdir
-        with open(fn, 'at', encoding='utf-8') as fslog:
+        with koji._open_text_file(fn, 'at') as fslog:
             logfile = "%s/do_mounts.log" % self.workdir
             uploadpath = self.getUploadDir()
             error = None
@@ -309,7 +309,7 @@ class RunRootTask(koji.tasks.BaseTaskHandler):
         mounts = set()
         fn = '%s/tmp/runroot_mounts' % rootdir
         if os.path.exists(fn):
-            with open(fn, 'rt', encoding='utf-8') as fslog:
+            with koji._open_text_file(fn) as fslog:
                 for line in fslog.readlines():
                     mounts.add(line.strip())
         # also, check /proc/mounts just in case

tests/test_cli/test_mock_config.py

@@ -333,7 +333,7 @@ config_opts['macros']['%distribution'] = 'Koji Testing'
         fobj = mock.MagicMock()
         openf.return_value.__enter__.return_value = fobj
         anon_handle_mock_config(options, session, arguments)
-        openf.assert_called_with('/tmp/mock.out', 'wt', encoding='utf-8')
+        openf.assert_called_once()
         fobj.write.assert_called_once_with(self.mock_output)
         gen_config_mock.assert_called_with(
             self.progname, arch, **opts)

tests/test_plugins/test_runroot_builder.py

@@ -13,14 +13,6 @@ __main__.BuildRoot = kojid.BuildRoot
 import koji
 import runroot
 
-def mock_open():
-    """Return the right patch decorator for open"""
-    if six.PY2:
-        return mock.patch('__builtin__.open')
-    else:
-        return mock.patch('builtins.open')
-
-
 if six.PY2:
     CONFIG_PARSER = 'six.moves.configparser.SafeConfigParser'
 else:
@@ -192,7 +184,7 @@ class TestMounts(unittest.TestCase):
         self.assertEqual(self.t._get_path_params('/mnt/archive', 'rw'),
             ('archive.org:/vol/archive/', '/mnt/archive', 'nfs', 'rw,hard,intr,nosuid,nodev,noatime,tcp'))
 
-    @mock_open()
+    @mock.patch('koji._open_text_file')
     @mock.patch('os.path.isdir')
     @mock.patch('runroot.log_output')
     def test_do_mounts(self, log_output, is_dir, open_mock):
@@ -293,7 +285,7 @@ class TestMounts(unittest.TestCase):
         self.t.do_mounts.assert_not_called()
 
 
-    @mock_open()
+    @mock.patch('koji._open_text_file')
     @mock.patch('runroot.scan_mounts')
     @mock.patch('os.unlink')
     @mock.patch('subprocess.Popen')

vm/kojikamid.py

@@ -331,7 +331,7 @@ class WindowsBuild(object):
                 raise BuildError('Unknown checksum type %s for %s' % (  # noqa: F821
                                  checksum_type,
                                  os.path.basename(fileinfo['localpath'])))
-        with open(destpath, 'wt', encoding='utf-8') as destfile:
+        with _open_text_file(destpath, 'wt') as destfile:
             offset = 0
             while True:
                 encoded = self.server.getFile(buildinfo, fileinfo, encode_int(offset), 1048576,
@@ -412,7 +412,7 @@ class WindowsBuild(object):
         """Do the build: run the execute line(s) with cmd.exe"""
         tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp', suffix='.bat',
                                           dir='/cygdrive/c/Windows/Temp')
-        script = os.fdopen(tmpfd, 'wt', encoding='utf-8')
+        script = _open_text_file(tmpfd, 'wt')
         for attr in ['source_dir', 'spec_dir', 'patches_dir']:
             val = getattr(self, attr)
             if val:
@@ -449,7 +449,7 @@ class WindowsBuild(object):
     def bashBuild(self):
         """Do the build: run the execute line(s) with bash"""
         tmpfd, tmpname = tempfile.mkstemp(prefix='koji-tmp.', dir='/tmp')
-        script = os.fdopen(tmpfd, 'wt', encoding='utf-8')
+        script = _open_text_file(tmpfd, 'wt')
         script.write("export source_dir='%s'\n" % self.source_dir)
         script.write("export spec_dir='%s'\n" % self.spec_dir)
         if self.patches_dir:
@@ -657,7 +657,7 @@ def setup_logging(opts):
     if opts.debug:
         level = logging.DEBUG
     logger.setLevel(level)
-    logfd = open(logfile, 'wt', encoding='utf-8')
+    logfd = _open_text_file(logfile, 'wt')
     handler = logging.StreamHandler(logfd)
     handler.setLevel(level)
     handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(name)s: %(message)s'))