From f134e8416d3e82532c4ee80ba311873860f39361 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Jun 2023 00:42:06 +0200 Subject: [PATCH] gh-104212: Add importlib.util.load_source_path() function --- Doc/library/importlib.rst | 14 +++ Doc/whatsnew/3.12.rst | 9 ++ Lib/importlib/util.py | 19 ++++ Lib/test/test_importlib/test_util.py | 98 +++++++++++++++++++ ...-06-14-00-54-11.gh-issue-104212.4IJaKp.rst | 2 + 5 files changed, 142 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-06-14-00-54-11.gh-issue-104212.4IJaKp.rst diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 65aaad0df9ee66..b127d6ca3f49de 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -1240,6 +1240,20 @@ an :term:`importer`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. +.. function:: load_source_path(module_name, filename) + + Load a module from a filename: execute the module and cache it to + :data:`sys.modules`. + + *module_name* must not contain dots. A package cannot be imported by its + directory path, whereas its ``__init__.py`` file (ex: + ``package/__init__.py``) can be imported. + + The module is always executed even if it's already cached in + :data:`sys.modules`. + + .. versionadded:: 3.12 + .. function:: source_hash(source_bytes) Return the hash of *source_bytes* as bytes. A hash-based ``.pyc`` file embeds diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 8febb6cc86b6fe..3860009c091c73 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -570,6 +570,12 @@ fractions * Objects of type :class:`fractions.Fraction` now support float-style formatting. (Contributed by Mark Dickinson in :gh:`100161`.) +importlib +--------- + +* Add :func:`importlib.util.load_source_path` to load a module from a filename. + (Contributed by Victor Stinner in :gh:`104212`.) + inspect ------- @@ -1371,6 +1377,9 @@ Removed * Replace ``imp.new_module(name)`` with ``types.ModuleType(name)``. + * Replace ``imp.load_source(module_name, filename)`` + with ``importlib.util.load_source_path(module_name, filename)``. + * Removed the ``suspicious`` rule from the documentation Makefile, and removed ``Doc/tools/rstlint.py``, both in favor of `sphinx-lint `_. diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index f4d6e82331516f..bb1eec038aa21c 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -10,6 +10,7 @@ from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache from ._bootstrap_external import spec_from_file_location +from ._bootstrap_external import SourceFileLoader import _imp import sys @@ -246,3 +247,21 @@ def exec_module(self, module): loader_state['__class__'] = module.__class__ module.__spec__.loader_state = loader_state module.__class__ = _LazyModule + + +def load_source_path(module_name, filename): + """Load a module from a filename.""" + if '.' in module_name: + raise ValueError(f"module name must not contain dots: {module_name!r}") + + loader = SourceFileLoader(module_name, filename) + # use spec_from_file_location() to always set the __file__ attribute, + # even if the filename does not end with ".py" + spec = spec_from_file_location(module_name, filename, + loader=loader, + submodule_search_locations=[]) + + module = module_from_spec(spec) + sys.modules[module.__name__] = module + loader.exec_module(module) + return module diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index e967adc9451c81..d7fb01389b5108 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -1,3 +1,4 @@ +from test.test_importlib import fixtures from test.test_importlib import util abc = util.import_importlib('importlib.abc') @@ -12,6 +13,7 @@ import string import sys from test import support +from test.support import import_helper, os_helper import textwrap import types import unittest @@ -758,5 +760,101 @@ def test_complete_multi_phase_init_module(self): self.run_with_own_gil(script) +class LoadSourcePathTests(unittest.TestCase): + def check_module(self, mod, modname, filename, is_package=False): + abs_filename = os.path.abspath(filename) + + self.assertIsInstance(mod, types.ModuleType) + self.assertEqual(mod.__name__, modname) + self.assertEqual(mod.__file__, abs_filename) + self.assertIn(modname, sys.modules) + self.assertIs(sys.modules[modname], mod) + self.assertEqual(mod.__path__, [os.path.dirname(abs_filename)]) + + loader = mod.__loader__ + self.assertEqual(loader.is_package(modname), is_package) + + spec = mod.__spec__ + self.assertEqual(spec.name, modname) + self.assertEqual(spec.origin, abs_filename) + + def test_filename(self): + modname = 'test_load_source_path_mod' + # Filename doesn't have to end with ".py" suffix + filename = 'load_source_path_filename' + side_effect = 'load_source_path_side_effect' + + def delete_side_effect(): + try: + delattr(sys, side_effect) + except AttributeError: + pass + + self.assertNotIn(modname, sys.modules) + self.addCleanup(import_helper.unload, modname) + + self.assertFalse(hasattr(sys, side_effect)) + self.addCleanup(delete_side_effect) + + # Use a temporary directory to remove __pycache__/ subdirectory + with fixtures.tempdir_as_cwd(): + with open(filename, "w", encoding="utf8") as fp: + print("attr = 'load_source_path_attr'", file=fp) + print(f"import sys; sys.{side_effect} = 1", file=fp) + + mod = importlib.util.load_source_path(modname, filename) + + self.check_module(mod, modname, filename) + self.assertEqual(mod.attr, 'load_source_path_attr') + self.assertEqual(getattr(sys, side_effect), 1) + + # reload cached in sys.modules: the module is executed again + self.assertIn(modname, sys.modules) + setattr(sys, side_effect, 0) + mod = importlib.util.load_source_path(modname, filename) + self.assertEqual(getattr(sys, side_effect), 1) + + # reload uncached in sys.modules: the module is executed again + del sys.modules[modname] + setattr(sys, side_effect, 0) + mod = importlib.util.load_source_path(modname, filename) + self.assertEqual(getattr(sys, side_effect), 1) + + def test_dots(self): + modname = 'package.submodule' + filename = __file__ + with self.assertRaises(ValueError) as cm: + importlib.util.load_source_path(modname, filename) + + err_msg = str(cm.exception) + self.assertIn("module name must not contain dots", err_msg) + self.assertIn(repr(modname), err_msg) + + def test_package(self): + modname = 'test_load_source_path_package' + dirname = 'load_source_path_dir' + filename = os.path.join('load_source_path_dir', '__init__.py') + + self.assertNotIn(modname, sys.modules) + self.addCleanup(import_helper.unload, modname) + + # Use a temporary directory to remove __pycache__/ subdirectory + with fixtures.tempdir_as_cwd(): + os.mkdir(dirname) + with open(filename, "w", encoding="utf8") as fp: + print("attr = 'load_source_path_pkg'", file=fp) + + # Package cannot be imported from a directory. It can with + # IsADirectoryError on Unix and PermissionError on Windows. + with self.assertRaises(OSError): + importlib.util.load_source_path(modname, dirname) + + # whereas loading a package __init__.py file is ok + mod = importlib.util.load_source_path(modname, filename) + + self.check_module(mod, modname, filename, is_package=True) + self.assertEqual(mod.attr, 'load_source_path_pkg') + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2023-06-14-00-54-11.gh-issue-104212.4IJaKp.rst b/Misc/NEWS.d/next/Library/2023-06-14-00-54-11.gh-issue-104212.4IJaKp.rst new file mode 100644 index 00000000000000..733ed10869ec14 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-14-00-54-11.gh-issue-104212.4IJaKp.rst @@ -0,0 +1,2 @@ +Add :func:`importlib.util.load_source_path` to load a module from a filename. +Patch by Victor Stinner.