Merge pull request #20611 from VadimLevin:dev/vlevin/pure-python-modules

* feat: OpenCV extension with pure Python modules

* feat: cv2 is now a Python package instead of extension module

Python package cv2 now can handle both Python and C extension modules
properly without additional "subfolders" like "_extra_py_code".

* feat: can call native function from its reimplementation in Python

modules/python/package/cv2/__init__.py

@@ -2,6 +2,7 @@
 OpenCV Python binary extension loader
 '''
 import os
+import importlib
 import sys
 
 __all__ = []
@@ -15,17 +16,54 @@ except ImportError:
     print('    pip install numpy')
     raise
 
-
-py_code_loader = None
-if sys.version_info[:2] >= (3, 0):
-    try:
-        from . import _extra_py_code as py_code_loader
-    except:
-        pass
-
 # TODO
 # is_x64 = sys.maxsize > 2**32
 
+
+def __load_extra_py_code_for_module(base, name, enable_debug_print=False):
+    module_name = "{}.{}".format(__name__, name)
+    export_module_name = "{}.{}".format(base, name)
+    native_module = sys.modules.pop(module_name, None)
+    try:
+        py_module = importlib.import_module(module_name)
+    except ImportError as err:
+        if enable_debug_print:
+            print("Can't load Python code for module:", module_name,
+                  ". Reason:", err)
+        # Extension doesn't contain extra py code
+        return False
+
+    if not hasattr(base, name):
+        setattr(sys.modules[base], name, py_module)
+    sys.modules[export_module_name] = py_module
+    # If it is C extension module it is already loaded by cv2 package
+    if native_module:
+        setattr(py_module, "_native", native_module)
+        for k, v in filter(lambda kv: not hasattr(py_module, kv[0]),
+                           native_module.__dict__.items()):
+            if enable_debug_print: print('    symbol: {} = {}'.format(k, v))
+            setattr(py_module, k, v)
+    return True
+
+
+def __collect_extra_submodules(enable_debug_print=False):
+    def modules_filter(module):
+        return all((
+             # module is not internal
+             not module.startswith("_"),
+             # it is not a file
+             os.path.isdir(os.path.join(_extra_submodules_init_path, module))
+        ))
+    if sys.version_info[0] < 3:
+        if enable_debug_print:
+            print("Extra submodules is loaded only for Python 3")
+        return []
+
+    __INIT_FILE_PATH = os.path.abspath(__file__)
+    _extra_submodules_init_path = os.path.dirname(__INIT_FILE_PATH)
+    return filter(modules_filter, os.listdir(_extra_submodules_init_path))
+
+
 def bootstrap():
     import sys
 
@@ -107,23 +145,36 @@ def bootstrap():
         # amending of LD_LIBRARY_PATH works for sub-processes only
         os.environ['LD_LIBRARY_PATH'] = ':'.join(l_vars['BINARIES_PATHS']) + ':' + os.environ.get('LD_LIBRARY_PATH', '')
 
-    if DEBUG: print('OpenCV loader: replacing cv2 module')
-    del sys.modules['cv2']
-    import cv2
+    if DEBUG: print("Relink everything from native cv2 module to cv2 package")
+
+    py_module = sys.modules.pop("cv2")
+
+    native_module = importlib.import_module("cv2")
+
+    sys.modules["cv2"] = py_module
+    setattr(py_module, "_native", native_module)
+
+    for item_name, item in filter(lambda kv: kv[0] not in ("__file__", "__loader__", "__spec__",
+                                                           "__name__", "__package__"),
+                                  native_module.__dict__.items()):
+        if item_name not in g_vars:
+            g_vars[item_name] = item
 
     sys.path = save_sys_path  # multiprocessing should start from bootstrap code (https://github.com/opencv/opencv/issues/18502)
 
     try:
-        import sys
         del sys.OpenCV_LOADER
-    except:
-        pass
+    except Exception as e:
+        if DEBUG:
+            print("Exception during delete OpenCV_LOADER:", e)
 
     if DEBUG: print('OpenCV loader: binary extension... OK')
 
-    if py_code_loader:
-        py_code_loader.init('cv2')
+    for submodule in __collect_extra_submodules(DEBUG):
+        if __load_extra_py_code_for_module("cv2", submodule, DEBUG):
+            if DEBUG: print("Extra Python code for", submodule, "is loaded")
 
     if DEBUG: print('OpenCV loader: DONE')
 
+
 bootstrap()

modules/python/package/cv2/_extra_py_code/__init__.py

@@ -1,53 +0,0 @@
-import sys
-import importlib
-
-__all__ = ['init']
-
-
-DEBUG = False
-if hasattr(sys, 'OpenCV_LOADER_DEBUG'):
-    DEBUG = True
-
-
-def _load_py_code(base, name):
-    try:
-        m = importlib.import_module(__name__ + name)
-    except ImportError:
-        return  # extension doesn't exist?
-
-    if DEBUG: print('OpenCV loader: added python code extension for: ' + name)
-
-    if hasattr(m, '__all__'):
-        export_members = { k : getattr(m, k) for k in m.__all__ }
-    else:
-        export_members = m.__dict__
-
-    for k, v in export_members.items():
-        if k.startswith('_'):  # skip internals
-            continue
-        if isinstance(v, type(sys)):  # don't bring modules
-            continue
-        if DEBUG: print('    symbol: {} = {}'.format(k, v))
-        setattr(sys.modules[base + name ], k, v)
-
-    del sys.modules[__name__ + name]
-
-
-# TODO: listdir
-def init(base):
-    _load_py_code(base, '.cv2')  # special case
-    prefix = base
-    prefix_len = len(prefix)
-
-    modules = [ m for m in sys.modules.keys() if m.startswith(prefix) ]
-    for m in modules:
-        m2 = m[prefix_len:]  # strip prefix
-        if len(m2) == 0:
-            continue
-        if m2.startswith('._'):  # skip internals
-            continue
-        if m2.startswith('.load_config_'):  # skip helper files
-            continue
-        _load_py_code(base, m2)
-
-    del sys.modules[__name__]

modules/python/package/extra_modules/misc/__init__.py

@@ -0,0 +1 @@
+from .version import get_ocv_version

modules/python/package/extra_modules/misc/version.py

@@ -0,0 +1,5 @@
+import cv2
+
+
+def get_ocv_version():
+    return getattr(cv2, "__version__", "unavailable")