importlib instead of imp

Fixes: https://pagure.io/koji/issue/2822

koji/__init__.py

@@ -28,7 +28,6 @@ import base64
 import datetime
 import errno
 import hashlib
-import imp
 import json
 import logging
 import logging.handlers
@@ -49,6 +48,13 @@ import warnings
 import weakref
 import xml.sax
 import xml.sax.handler
+try:
+    import importlib
+    import importlib.machinery
+except ImportError:  # pragma: no cover
+    # importlib not available for PY2, so we fall back to using imp
+    import imp as imp
+    importlib = None
 from fnmatch import fnmatch
 
 import dateutil.parser
@@ -2030,18 +2036,24 @@ def get_profile_module(profile_name, config=None):
     # Prepare module name
     mod_name = "__%s__%s" % (__name__, profile_name)
 
-    imp.acquire_lock()
+    if not importlib:
+        imp.acquire_lock()
+
     try:
         # Check if profile module exists and if so return it
         if mod_name in PROFILE_MODULES:
             return PROFILE_MODULES[mod_name]
 
         # Load current module under a new name
-        koji_module_loc = imp.find_module(__name__)
-        mod = imp.load_module(mod_name,
-                              None,
-                              koji_module_loc[1],
-                              koji_module_loc[2])
+        if importlib:
+            koji_spec = importlib.util.find_spec(__name__)
+            mod_spec = importlib.util.spec_from_file_location(mod_name, koji_spec.origin)
+            mod = importlib.util.module_from_spec(mod_spec)
+            sys.modules[mod_name] = mod
+            mod_spec.loader.exec_module(mod)
+        else:
+            koji_module_loc = imp.find_module(__name__)
+            mod = imp.load_module(mod_name, None, koji_module_loc[1], koji_module_loc[2])
 
         # Tweak config of the new module
         mod.config = config
@@ -2053,7 +2065,8 @@ def get_profile_module(profile_name, config=None):
 
         PROFILE_MODULES[mod_name] = mod
     finally:
-        imp.release_lock()
+        if not importlib:
+            imp.release_lock()
 
     return mod
 

koji/plugin.py

@@ -21,7 +21,6 @@
 
 from __future__ import absolute_import
 
-import imp
 import logging
 import sys
 import traceback
@@ -31,6 +30,14 @@ import six
 import koji
 from koji.util import encode_datetime_recurse
 
+try:
+    import importlib
+    import importlib.machinery
+except ImportError:  # pragma: no cover
+    # importlib not available for PY2, so we fall back to using imp
+    import imp as imp
+    importlib = None
+
 # the available callback hooks and a list
 # of functions to be called for each event
 callbacks = {
@@ -86,15 +93,24 @@ class PluginTracker(object):
             path = self.searchpath
         if path is None:
             raise koji.PluginError("empty module search path")
-        file, pathname, description = imp.find_module(name, self.pathlist(path))
+        file = None
         try:
-            plugin = imp.load_module(mod_name, file, pathname, description)
+            if importlib:
+                orig_spec = importlib.machinery.PathFinder().find_spec(name, self.pathlist(path))
+                plugin_spec = importlib.util.spec_from_file_location(mod_name, orig_spec.origin)
+                plugin = importlib.util.module_from_spec(plugin_spec)
+                sys.modules[mod_name] = plugin
+                plugin_spec.loader.exec_module(plugin)
+            else:
+                file, pathname, description = imp.find_module(name, self.pathlist(path))
+                plugin = imp.load_module(mod_name, file, pathname, description)
         except Exception:
             msg = 'Loading plugin %s failed' % name
             logging.getLogger('koji.plugin').error(msg)
             raise
         finally:
-            file.close()
+            if file:
+                file.close()
         self.plugins[name] = plugin
         return plugin
 

tests/test_cli/loadcli.py

@@ -1,19 +1,26 @@
 from __future__ import absolute_import
 import os
-#import sys
+import sys
+
+try:
+    import importlib
+    import importlib.machinery
+except ImportError:
+    import imp
+    importlib = None
+
+CLI_MOD = "koji_cli_fake"
 
 # We have to do this craziness because 'import koji' is ambiguous.  Is it the
 # koji module, or the koji cli module.  Jump through hoops accordingly.
 # https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
 CLI_FILENAME = os.path.dirname(__file__) + "/../../cli/koji"
-'''
-if sys.version_info[0] >= 3:
-    import importlib.util
-    spec = importlib.util.spec_from_file_location("koji_cli", CLI_FILENAME)
+if importlib:
+    importlib.machinery.SOURCE_SUFFIXES.append('')
+    spec = importlib.util.spec_from_file_location(CLI_MOD, CLI_FILENAME)
     cli = importlib.util.module_from_spec(spec)
+    sys.modules[CLI_MOD] = cli
     spec.loader.exec_module(cli)
+    importlib.machinery.SOURCE_SUFFIXES.pop()
 else:
-'''
-
-import imp
-cli = imp.load_source('koji_cli_fake', CLI_FILENAME)
+    cli = imp.load_source(CLI_MOD, CLI_FILENAME)

tests/test_docs_version.py

@@ -3,14 +3,25 @@ import os
 import six
 import subprocess
 import unittest
+import sys
+
+try:
+    import importlib.util
+except ImportError:  # pragma: no cover
+    import imp as imp
+    importlib = None
 
 # docs version lives in docs/source/conf.py
 TOPDIR = os.path.dirname(__file__) + '/..'
 SPHINX_CONF = TOPDIR + '/docs/source/conf.py'
 
-import imp
-import os
-sphinx_conf = imp.load_source('sphinx_conf', SPHINX_CONF)
+if importlib:
+    spec = importlib.util.spec_from_file_location("sphinx_conf", SPHINX_CONF)
+    sphinx_conf = importlib.util.module_from_spec(spec)
+    sys.modules["sphinx_conf"] = sphinx_conf
+    spec.loader.exec_module(sphinx_conf)
+else:
+    sphinx_conf = imp.load_source('sphinx_conf', SPHINX_CONF)
 
 
 class TestDocsVersion(unittest.TestCase):
@@ -35,4 +46,3 @@ class TestDocsVersion(unittest.TestCase):
         # docs 'version' is x.y instead of x.y.z
         dver = '.'.join(koji_version.split('.')[:-1])
         self.assertEqual(dver, sphinx_conf.version)
-

tests/test_kojira/loadkojira.py

@@ -1,8 +1,24 @@
 from __future__ import absolute_import
 import os
+import sys
 
+try:
+    import importlib
+    import importlib.machinery
+except ImportError:
+    import imp
+    importlib = None
 
 # TODO - libify kojira so we don't need this hack
-CLI_FILENAME = os.path.dirname(__file__) + "/../../util/kojira"
-import imp
-kojira = imp.load_source('kojira', CLI_FILENAME)
+KOJIRA_MOD = "kojira_"
+KOJIRA_FILENAME = os.path.dirname(__file__) + "/../../util/kojira"
+
+if importlib:
+    importlib.machinery.SOURCE_SUFFIXES.append('')
+    spec = importlib.util.spec_from_file_location(KOJIRA_MOD, KOJIRA_FILENAME)
+    kojira = importlib.util.module_from_spec(spec)
+    sys.modules[KOJIRA_MOD] = kojira
+    spec.loader.exec_module(kojira)
+    importlib.machinery.SOURCE_SUFFIXES.pop()
+else:
+    kojira = imp.load_source(KOJIRA_MOD, KOJIRA_FILENAME)

tests/test_lib/test_plugin.py

@@ -3,8 +3,16 @@ import copy
 import datetime
 import mock
 from six.moves import range
+import sys
 import unittest
 
+try:
+    import importlib
+    imp = None
+except ImportError:
+    import imp
+    importlib = None
+
 import koji
 import koji.util
 import koji.plugin
@@ -14,7 +22,7 @@ class TestCallbackDecorators(unittest.TestCase):
 
     def test_callback_decorator(self):
         def myfunc(a, b, c=None):
-            return [a,b,c]
+            return [a, b, c]
         callbacks = ('preImport', 'postImport')
         newfunc = koji.plugin.callback(*callbacks)(myfunc)
         self.assertEqual(newfunc.callbacks, callbacks)
@@ -23,7 +31,7 @@ class TestCallbackDecorators(unittest.TestCase):
 
     def test_ignore_error_decorator(self):
         def myfunc(a, b, c=None):
-            return [a,b,c]
+            return [a, b, c]
         newfunc = koji.plugin.ignore_error(myfunc)
         self.assertEqual(newfunc.failure_is_an_option, True)
         self.assertEqual(newfunc(1, 2), [1, 2, None])
@@ -31,7 +39,7 @@ class TestCallbackDecorators(unittest.TestCase):
 
     def test_export_decorator(self):
         def myfunc(a, b, c=None):
-            return [a,b,c]
+            return [a, b, c]
         newfunc = koji.plugin.export(myfunc)
         self.assertEqual(newfunc.exported, True)
         self.assertEqual(newfunc(1, 2), [1, 2, None])
@@ -39,7 +47,7 @@ class TestCallbackDecorators(unittest.TestCase):
 
     def test_export_cli_decorator(self):
         def myfunc(a, b, c=None):
-            return [a,b,c]
+            return [a, b, c]
         newfunc = koji.plugin.export_cli(myfunc)
         self.assertEqual(newfunc.exported_cli, True)
         self.assertEqual(newfunc(1, 2), [1, 2, None])
@@ -47,7 +55,7 @@ class TestCallbackDecorators(unittest.TestCase):
 
     def test_export_as_decorator(self):
         def myfunc(a, b, c=None):
-            return [a,b,c]
+            return [a, b, c]
         alias = "ALIAS"
         newfunc = koji.plugin.export_as(alias)(myfunc)
         self.assertEqual(newfunc.exported, True)
@@ -57,7 +65,7 @@ class TestCallbackDecorators(unittest.TestCase):
 
     def test_export_in_decorator_with_alias(self):
         def myfunc(a, b, c=None):
-            return [a,b,c]
+            return [a, b, c]
         newfunc = koji.plugin.export_in('MODULE', 'ALIAS')(myfunc)
         self.assertEqual(newfunc.exported, True)
         self.assertEqual(newfunc.export_alias, 'MODULE.ALIAS')
@@ -67,7 +75,7 @@ class TestCallbackDecorators(unittest.TestCase):
 
     def test_export_in_decorator_no_alias(self):
         def myfunc(a, b, c=None):
-            return [a,b,c]
+            return [a, b, c]
         newfunc = koji.plugin.export_in('MODULE')(myfunc)
         self.assertEqual(newfunc.exported, True)
         self.assertEqual(newfunc.export_alias, 'MODULE.myfunc')
@@ -141,8 +149,8 @@ class TestCallbacks(unittest.TestCase):
 
     def test_datetime_callback(self):
         dt1 = datetime.datetime.now()
-        dt2 = datetime.datetime(2001,1,1)
-        args = (dt1,"2",["three"], {4: dt2},)
+        dt2 = datetime.datetime(2001, 1, 1)
+        args = (dt1, "2", ["three"], {4: dt2},)
         kwargs = {'foo': [dt1, dt2]}
         cbtype = 'preTag'
         koji.plugin.register_callback(cbtype, self.datetime_callback)
@@ -154,8 +162,8 @@ class TestCallbacks(unittest.TestCase):
 
     def test_multiple_datetime_callback(self):
         dt1 = datetime.datetime.now()
-        dt2 = datetime.datetime(2001,1,1)
-        args = (dt1,"2",["three"], {4: dt2},)
+        dt2 = datetime.datetime(2001, 1, 1)
+        args = (dt1, "2", ["three"], {4: dt2},)
         kwargs = {'foo': [dt1, dt2]}
         cbtype = 'preTag'
         koji.plugin.register_callback(cbtype, self.datetime_callback)
@@ -195,70 +203,105 @@ class TestCallbacks(unittest.TestCase):
 class TestPluginTracker(unittest.TestCase):
 
     def setUp(self):
-        self.find_module = mock.patch('imp.find_module').start()
-        self.modfile = mock.MagicMock()
-        self.modpath = mock.MagicMock()
-        self.moddesc = mock.MagicMock()
-        self.find_module.return_value = (self.modfile, self.modpath,
-                self.moddesc)
-        self.load_module = mock.patch('imp.load_module').start()
+        self.tracker = koji.plugin.PluginTracker(path='/MODPATH')
+        if imp:
+            self.modfile = mock.MagicMock()
+            self.modpath = mock.MagicMock()
+            self.moddesc = mock.MagicMock()
+            self.find_module = mock.patch('imp.find_module').start()
+
+            self.find_module.return_value = (self.modfile, self.modpath, self.moddesc)
+
+            self.load_module = mock.patch('imp.load_module').start()
+        else:
+            self.finder = mock.patch('importlib.machinery.PathFinder').start()
+            self.find_spec = self.finder.return_value.find_spec
+            self.find_spec.return_value = mock.MagicMock()
+            self.spec_from_file_location = mock.patch(
+                'importlib.util.spec_from_file_location').start()
+            self.module_from_spec = mock.patch('importlib.util.module_from_spec').start()
 
     def tearDown(self):
         mock.patch.stopall()
+        for key in ['hello', 'hey', 'dup_module']:
+            if '_koji_plugin__' + key in sys.modules:
+                del sys.modules['_koji_plugin__' + key]
+        del self.tracker
+
+    def _set_returned_module(self, mod):
+        if imp:
+            self.load_module.return_value = mod
+        else:
+            self.module_from_spec.return_value = mod
+
+    def _set_module_side_effect(self, se):
+        if imp:
+            self.load_module.side_effect = se
+        else:
+            self.module_from_spec.side_effect = se
 
     def test_tracked_plugin(self):
-        tracker = koji.plugin.PluginTracker(path='/MODPATH')
-        self.load_module.return_value = 'MODULE!'
-        tracker.load('hello')
-        self.assertEqual(tracker.get('hello'), 'MODULE!')
-        self.find_module.assert_called_once_with('hello', ['/MODPATH'])
+        self._set_returned_module('MODULE!')
+        self.tracker.load('hello')
+        self.assertEqual(self.tracker.get('hello'), 'MODULE!')
+        if imp:
+            self.find_module.assert_called_once_with('hello', ['/MODPATH'])
+        else:
+            self.find_spec.assert_called_once_with('hello', ['/MODPATH'])
 
     def test_plugin_reload(self):
-        tracker = koji.plugin.PluginTracker(path='/MODPATH')
-        self.load_module.return_value = 'MODULE!'
-        tracker.load('hello')
-        self.assertEqual(tracker.get('hello'), 'MODULE!')
+        self._set_returned_module('MODULE!')
+        self.tracker.load('hello')
+        self.assertEqual(self.tracker.get('hello'), 'MODULE!')
 
         # should not reload if we don't ask
-        self.load_module.return_value = 'DUPLICATE!'
-        tracker.load('hello')
-        self.assertEqual(tracker.get('hello'), 'MODULE!')
+        self._set_returned_module('DUPLICATE!')
+        self.tracker.load('hello')
+        self.assertEqual(self.tracker.get('hello'), 'MODULE!')
 
         # should reload if we do ask
-        tracker.load('hello', reload=True)
-        self.assertEqual(tracker.get('hello'), 'DUPLICATE!')
+        self.tracker.load('hello', reload=True)
+        self.assertEqual(self.tracker.get('hello'), 'DUPLICATE!')
 
         # should throw exception if not reloading and duplicate in sys.modules
         module_mock = mock.MagicMock()
         with mock.patch.dict('sys.modules', _koji_plugin__dup_module=module_mock):
             with self.assertRaises(koji.PluginError):
-                tracker.load('dup_module')
+                self.tracker.load('dup_module')
 
     def test_no_plugin_path(self):
         tracker = koji.plugin.PluginTracker()
         with self.assertRaises(koji.PluginError):
             tracker.load('hello')
-        self.load_module.assert_not_called()
+        if imp:
+            self.load_module.assert_not_called()
+        else:
+            self.module_from_spec.assert_not_called()
         self.assertEqual(tracker.get('hello'), None)
 
     def test_plugin_path_list(self):
-        tracker = koji.plugin.PluginTracker(path='/MODPATH')
-        self.load_module.return_value = 'MODULE!'
-        tracker.load('hello', path=['/PATH1', '/PATH2'])
-        self.assertEqual(tracker.get('hello'), 'MODULE!')
-        self.find_module.assert_called_once_with('hello', ['/PATH1', '/PATH2'])
+        self._set_returned_module('MODULE!')
+        self.tracker.load('hello', path=['/PATH1', '/PATH2'])
+        self.assertEqual(self.tracker.get('hello'), 'MODULE!')
+        if imp:
+            self.find_module.assert_called_once_with('hello', ['/PATH1', '/PATH2'])
+            self.find_module.reset_mock()
+        else:
+            self.find_spec.assert_called_once_with('hello', ['/PATH1', '/PATH2'])
+            self.find_spec.reset_mock()
 
-        self.find_module.reset_mock()
-        tracker.load('hey', path='/PATH1')
-        self.assertEqual(tracker.get('hey'), 'MODULE!')
-        self.find_module.assert_called_once_with('hey', ['/PATH1'])
+        self.tracker.load('hey', path='/PATH1')
+        self.assertEqual(self.tracker.get('hey'), 'MODULE!')
+        if imp:
+            self.find_module.assert_called_once_with('hey', ['/PATH1'])
+        else:
+            self.find_spec.assert_called_once_with('hey', ['/PATH1'])
 
     @mock.patch('logging.getLogger')
     def test_bad_plugin(self, getLogger):
-        tracker = koji.plugin.PluginTracker(path='/MODPATH')
-        self.load_module.side_effect = TestError
+        self._set_module_side_effect(TestError)
         with self.assertRaises(TestError):
-            tracker.load('hello')
-        self.assertEqual(tracker.get('hello'), None)
+            self.tracker.load('hello')
+        self.assertEqual(self.tracker.get('hello'), None)
         getLogger.assert_called_once()
         getLogger.return_value.error.assert_called_once()

tests/test_www/loadwebindex.py

@@ -1,9 +1,20 @@
 import os
-import imp
+import sys
 
-# We have to do this craziness because 'import koji' is ambiguous.  Is it the
-# koji module, or the koji cli module.  Jump through hoops accordingly.
-# https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
+try:
+    import importlib
+    import importlib.machinery
+except ImportError:
+    import imp
+    importlib = None
+
+INDEX_MOD = "index_fake"
 INDEX_FILENAME = os.path.dirname(__file__) + "/../../www/kojiweb/index.py"
 
-webidx = imp.load_source('index_fake', INDEX_FILENAME)
+if importlib:
+    spec = importlib.util.spec_from_file_location(INDEX_MOD, INDEX_FILENAME)
+    webidx = importlib.util.module_from_spec(spec)
+    sys.modules[INDEX_MOD] = webidx
+    spec.loader.exec_module(webidx)
+else:
+    cli = imp.load_source(INDEX_MOD, INDEX_FILENAME)