diff --git a/CMakeLists.txt b/CMakeLists.txt index c7c19158..3565a663 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -171,6 +171,7 @@ if(WANT_PERL) find_package(PerlLibs 5.10 REQUIRED) endif() if (WANT_PYTHON) + set (_MIN_PYTHON_VERSION 3.4) find_package(Perl 5.10 REQUIRED) # VERSION_GREATER_EQUAL is available only since 3.7 if (CMAKE_VERSION VERSION_LESS 3.12) @@ -196,8 +197,10 @@ if (WANT_PYTHON) message(FATAL_ERROR "Invalid value of WANT_PYTHON_VERSION") endif() endif() - if (Python3_FOUND AND Python3_VERSION VERSION_LESS 3.3) - message(STATUS "Python too old, need at least 3.3") + if (Python3_FOUND AND Python3_VERSION VERSION_LESS + ${_MIN_PYTHON_VERSION}) + message(STATUS + "Python too old, need at least ${_MIN_PYTHON_VERSION}") set(Python3_FOUND OFF) else() # Compatibility with pkg-config variables @@ -206,9 +209,11 @@ if (WANT_PYTHON) endif() if (NOT Python3_FOUND AND WANT_PYTHON_VERSION MATCHES "^python") # Since python 3.8, -embed is required for embedding. - pkg_check_modules(Python3 "${WANT_PYTHON_VERSION}-embed >= 3.3") + pkg_check_modules(Python3 + "${WANT_PYTHON_VERSION}-embed >= ${_MIN_PYTHON_VERSION}") if (NOT Python3_FOUND) - pkg_check_modules(Python3 "${WANT_PYTHON_VERSION} >= 3.3") + pkg_check_modules(Python3 + "${WANT_PYTHON_VERSION} >= ${_MIN_PYTHON_VERSION}") endif() endif() if (NOT Python3_FOUND) diff --git a/modules/modpython/znc.py b/modules/modpython/znc.py index 478c5a39..feb98fda 100644 --- a/modules/modpython/znc.py +++ b/modules/modpython/znc.py @@ -22,10 +22,13 @@ if os.environ.get('ZNC_MODPYTHON_COVERAGE'): _cov.start() from functools import wraps -import imp -import re -import traceback import collections.abc +import importlib.abc +import importlib.machinery +import importlib.util +import re +import sys +import traceback from znc_core import * @@ -687,48 +690,86 @@ make_inherit(Module, CPyModule, '_cmod') make_inherit(Timer, CPyTimer, '_ctimer') -def find_open(modname): - '''Returns (pymodule, datapath)''' - for d in CModules.GetModDirs(): - # d == (libdir, datadir) - try: - x = imp.find_module(modname, [d[0]]) - except ImportError: - # no such file in dir d - continue - # x == (, - # './modules/admin.so', ('.so', 'rb', 3)) - # x == (, './modules/pythontest.py', ('.py', 'U', 1)) - if x[0] is None and x[2][2] != imp.PKG_DIRECTORY: - # the same - continue - if x[2][0] == '.so': - try: - pymodule = imp.load_module(modname, *x) - except ImportError: - # found needed .so but can't load it... - # maybe it's normal (non-python) znc module? - # another option here could be to "continue" - # search of python module in other moddirs. - # but... we respect C++ modules ;) - return (None, None) - finally: - x[0].close() - else: - # this is not .so, so it can be only python module .py or .pyc - try: - pymodule = imp.load_module(modname, *x) - finally: - if x[0]: - x[0].close() - return (pymodule, d[1]+modname) - else: - # nothing found - return (None, None) +class ZNCModuleLoader(importlib.abc.SourceLoader): + def __init__(self, modname, pypath): + self.pypath = pypath + + def create_module(self, spec): + self._datadir = spec.loader_state[0] + self._package_dir = spec.loader_state[1] + return super().create_module(spec) + + def get_data(self, path): + with open(path, 'rb') as f: + return f.read() + + def get_filename(self, fullname): + return self.pypath + + +class ZNCModuleFinder(importlib.abc.MetaPathFinder): + @staticmethod + def find_spec(fullname, path, target=None): + if fullname == 'znc_modules': + spec = importlib.util.spec_from_loader(fullname, None, is_package=True) + return spec + parts = fullname.split('.') + if parts[0] != 'znc_modules': + return + def dirs(): + if len(parts) == 2: + # common case + yield from CModules.GetModDirs() + else: + # the module is a package and tries to load a submodule of it + for libdir in sys.modules['znc_modules.' + parts[1]].__loader__._package_dir: + yield libdir, None + for libdir, datadir in dirs(): + finder = importlib.machinery.FileFinder(libdir, + (ZNCModuleLoader, importlib.machinery.SOURCE_SUFFIXES)) + spec = finder.find_spec('.'.join(parts[1:])) + if spec: + spec.name = fullname + spec.loader_state = (datadir, spec.submodule_search_locations) + # It almost works with original submodule_search_locations, + # then python will find submodules of the package itself, + # without calling out to ZNCModuleFinder or ZNCModuleLoader. + # But updatemod will be flaky for those submodules because as + # of py3.8 importlib.invalidate_caches() goes only through + # sys.meta_path, but not sys.path_hooks. So we make them load + # through ZNCModuleFinder too, but still remember the original + # dir so that the whole module comes from a single entry in + # CModules.GetModDirs(). + spec.submodule_search_locations = [] + return spec + + +sys.meta_path.append(ZNCModuleFinder()) _py_modules = set() +def find_open(modname): + '''Returns (pymodule, datapath)''' + fullname = 'znc_modules.' + modname + for m in _py_modules: + if m.GetModName() == modname: + break + else: + # module is not loaded, clean up previous attempts to load it or even + # to list as available modules + # This is to to let updatemod work + to_remove = [] + for m in sys.modules: + if m == fullname or m.startswith(fullname + '.'): + to_remove.append(m) + for m in to_remove: + del sys.modules[m] + try: + module = importlib.import_module(fullname) + except ImportError: + return (None, None) + return (module, os.path.join(module.__loader__._datadir, modname)) + def load_module(modname, args, module_type, user, network, retmsg, modpython): '''Returns 0 if not found, 1 on loading error, 2 on success''' if re.search(r'[^a-zA-Z0-9_]', modname) is not None: diff --git a/test/integration/tests/scripting.cpp b/test/integration/tests/scripting.cpp index ddd50f13..e4dcfc8e 100644 --- a/test/integration/tests/scripting.cpp +++ b/test/integration/tests/scripting.cpp @@ -243,5 +243,41 @@ TEST_F(ZNCTest, ModperlNV) { client.ReadUntil(":a b"); } +TEST_F(ZNCTest, ModpythonPackage) { + if (QProcessEnvironment::systemEnvironment().value( + "DISABLED_ZNC_PERL_PYTHON_TEST") == "1") { + return; + } + auto znc = Run(); + znc->CanLeak(); + + QDir dir(m_dir.path()); + ASSERT_TRUE(dir.mkpath("modules")); + ASSERT_TRUE(dir.cd("modules")); + ASSERT_TRUE(dir.mkpath("packagetest")); + InstallModule("packagetest/__init__.py", R"( + import znc + from .data import value + + class packagetest(znc.Module): + def OnModCommand(self, cmd): + self.PutModule('value = ' + value) + )"); + InstallModule("packagetest/data.py", "value = 'a'"); + + auto ircd = ConnectIRCd(); + auto client = LoginClient(); + client.Write("znc loadmod modpython"); + client.Write("znc loadmod packagetest"); + client.Write("PRIVMSG *packagetest :foo"); + client.ReadUntil("value = a"); + InstallModule("packagetest/data.py", "value = 'b'"); + client.Write("PRIVMSG *packagetest :foo"); + client.ReadUntil("value = a"); + client.Write("znc updatemod packagetest"); + client.Write("PRIVMSG *packagetest :foo"); + client.ReadUntil("value = b"); +} + } // namespace } // namespace znc_inttest