diff --git a/cmake/OpenCVDetectPython.cmake b/cmake/OpenCVDetectPython.cmake index 6e7bb18c1b..c93eb9f9a7 100644 --- a/cmake/OpenCVDetectPython.cmake +++ b/cmake/OpenCVDetectPython.cmake @@ -297,3 +297,10 @@ elseif(PYTHON3_EXECUTABLE AND PYTHON3INTERP_FOUND) set(PYTHON_DEFAULT_AVAILABLE "TRUE") set(PYTHON_DEFAULT_EXECUTABLE "${PYTHON3_EXECUTABLE}") endif() + +if(PYTHON_DEFAULT_AVAILABLE) + execute_process(COMMAND ${PYTHON_DEFAULT_EXECUTABLE} --version + OUTPUT_VARIABLE PYTHON_DEFAULT_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REGEX MATCH "[0-9]+.[0-9]+.[0-9]+" PYTHON_DEFAULT_VERSION "${PYTHON_DEFAULT_VERSION}") +endif() diff --git a/modules/python/bindings/CMakeLists.txt b/modules/python/bindings/CMakeLists.txt index b39c679615..918411864c 100644 --- a/modules/python/bindings/CMakeLists.txt +++ b/modules/python/bindings/CMakeLists.txt @@ -76,11 +76,14 @@ set(cv2_generated_files string(REPLACE ";" "\n" opencv_hdrs_ "${opencv_hdrs}") file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/headers.txt" "${opencv_hdrs_}") +file(GLOB_RECURSE typing_stubs_generation_files "${PYTHON_SOURCE_DIR}/src2/typing_stubs_generation/*.py") add_custom_command( OUTPUT ${cv2_generated_files} COMMAND "${PYTHON_DEFAULT_EXECUTABLE}" "${PYTHON_SOURCE_DIR}/src2/gen2.py" "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/headers.txt" DEPENDS "${PYTHON_SOURCE_DIR}/src2/gen2.py" "${PYTHON_SOURCE_DIR}/src2/hdr_parser.py" + "${typing_stubs_generation_files}" + "${PYTHON_SOURCE_DIR}/src2/typing_stubs_generator.py" # not a real build dependency (file(WRITE) result): ${CMAKE_CURRENT_BINARY_DIR}/headers.txt ${opencv_hdrs} COMMENT "Generate files for Python bindings and documentation" @@ -88,6 +91,10 @@ add_custom_command( add_custom_target(gen_opencv_python_source DEPENDS ${cv2_generated_files}) +if(TARGET copy_opencv_typing_stubs) + add_dependencies(copy_opencv_typing_stubs gen_opencv_python_source) +endif() + set(cv2_custom_hdr "${CMAKE_CURRENT_BINARY_DIR}/pyopencv_custom_headers.h") set(cv2_custom_hdr_str "//user-defined headers\n") foreach(uh ${opencv_userdef_hdrs}) diff --git a/modules/python/common.cmake b/modules/python/common.cmake index c5df8bca8f..a233fe0232 100644 --- a/modules/python/common.cmake +++ b/modules/python/common.cmake @@ -34,6 +34,11 @@ if(TARGET gen_opencv_python_source) add_dependencies(${the_module} gen_opencv_python_source) endif() +if(TARGET copy_opencv_typing_stubs) + # Python 3.6+ + add_dependencies(${the_module} copy_opencv_typing_stubs) +endif() + ocv_assert(${PYTHON}_VERSION_MAJOR) ocv_assert(${PYTHON}_VERSION_MINOR) diff --git a/modules/python/package/setup.py b/modules/python/package/setup.py index 13fb4e7b14..aa856ec7a5 100644 --- a/modules/python/package/setup.py +++ b/modules/python/package/setup.py @@ -1,9 +1,19 @@ import os -import sys -import platform import setuptools -SCRIPT_DIR=os.path.dirname(os.path.abspath(__file__)) + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def collect_module_typing_stub_files(root_module_path): + stub_files = [] + for module_path, _, files in os.walk(root_module_path): + stub_files.extend( + map(lambda p: os.path.join(module_path, p), + filter(lambda f: f.endswith(".pyi"), files)) + ) + return stub_files + def main(): os.chdir(SCRIPT_DIR) @@ -13,6 +23,13 @@ def main(): long_description = 'Open Source Computer Vision Library Python bindings' # TODO + root_module_path = os.path.join(SCRIPT_DIR, "cv2") + py_typed_path = os.path.join(root_module_path, "py.typed") + if os.path.isfile(py_typed_path): + typing_stub_files = collect_module_typing_stub_files(root_module_path) + if len(typing_stub_files) > 0: + typing_stub_files.append(py_typed_path) + setuptools.setup( name=package_name, version=package_version, @@ -22,6 +39,9 @@ def main(): long_description=long_description, long_description_content_type="text/markdown", packages=setuptools.find_packages(), + package_data={ + "cv2": typing_stub_files + }, maintainer="OpenCV Team", install_requires="numpy", classifiers=[ @@ -55,5 +75,6 @@ def main(): ], ) + if __name__ == '__main__': main() diff --git a/modules/python/python_loader.cmake b/modules/python/python_loader.cmake index 670d85154c..4226b82736 100644 --- a/modules/python/python_loader.cmake +++ b/modules/python/python_loader.cmake @@ -45,8 +45,6 @@ foreach(fname ${PYTHON_LOADER_FILES}) endif() endforeach() - - if(WIN32) if(CMAKE_GENERATOR MATCHES "Visual Studio") list(APPEND CMAKE_PYTHON_BINARIES_PATH "'${EXECUTABLE_OUTPUT_PATH}/Release'") # TODO: CMAKE_BUILD_TYPE is not defined @@ -126,3 +124,20 @@ if(NOT "${OPENCV_PYTHON_EXTRA_MODULES_PATH}" STREQUAL "") ocv_add_python_files_from_path(${extra_ocv_py_modules_path}) endforeach() endif() + +if(${PYTHON}_VERSION_STRING VERSION_GREATER "3.6" AND PYTHON_DEFAULT_VERSION VERSION_GREATER "3.6") + add_custom_target(copy_opencv_typing_stubs) + # Copy all generated stub files to python_loader directory only if + # generation succeeds, this behvoir can't be achieved with default + # CMake constructions, because failed generation produces a warning instead of + # halts on hard error. + add_custom_command( + TARGET copy_opencv_typing_stubs + COMMAND ${PYTHON_DEFAULT_EXECUTABLE} ${PYTHON_SOURCE_DIR}/src2/copy_typings_stubs_on_success.py + --stubs_dir ${OPENCV_PYTHON_BINDINGS_DIR}/cv2 + --output_dir ${__loader_path}/cv2 + ) + if(DEFINED OPENCV_PYTHON_INSTALL_PATH) + install(DIRECTORY "${OPENCV_PYTHON_BINDINGS_DIR}/cv2" DESTINATION "${OPENCV_PYTHON_INSTALL_PATH}" COMPONENT python) + endif() +endif() diff --git a/modules/python/src2/copy_typings_stubs_on_success.py b/modules/python/src2/copy_typings_stubs_on_success.py new file mode 100644 index 0000000000..efcf84ee82 --- /dev/null +++ b/modules/python/src2/copy_typings_stubs_on_success.py @@ -0,0 +1,45 @@ +import argparse +import warnings +import os +import sys + +if sys.version_info >= (3, 8, ): + # shutil.copytree received the `dirs_exist_ok` parameter + from functools import partial + import shutil + + copy_tree = partial(shutil.copytree, dirs_exist_ok=True) +else: + from distutils.dir_util import copy_tree + + +def main(): + args = parse_arguments() + py_typed_path = os.path.join(args.stubs_dir, 'py.typed') + if not os.path.isfile(py_typed_path): + warnings.warn( + '{} is missing, it means that typings stubs generation is either ' + 'failed or has been skipped. Ensure that Python 3.6+ is used for ' + 'build and there is no warnings during Python source code ' + 'generation phase.'.format(py_typed_path) + ) + return + copy_tree(args.stubs_dir, args.output_dir) + + +def parse_arguments(): + parser = argparse.ArgumentParser( + description='Copies generated typing stubs only when generation ' + 'succeeded. This is identified by presence of the `py.typed` file ' + 'inside typing stubs directory.' + ) + parser.add_argument('--stubs_dir', type=str, + help='Path to directory containing generated typing ' + 'stubs file') + parser.add_argument('--output_dir', type=str, + help='Path to output directory') + return parser.parse_args() + + +if __name__ == '__main__': + main() diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index 23f31c3e19..0ba643f12b 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -1,16 +1,34 @@ #!/usr/bin/env python from __future__ import print_function -import hdr_parser, sys, re, os +import hdr_parser, sys, re from string import Template -from pprint import pprint from collections import namedtuple +from itertools import chain + +from typing_stubs_generator import TypingStubsGenerator if sys.version_info[0] >= 3: from io import StringIO + else: from cStringIO import StringIO +if sys.version_info >= (3, 6): + from typing_stubs_generation import SymbolName +else: + SymbolName = namedtuple('SymbolName', ('namespaces', 'classes', 'name')) + + def parse_symbol_name(cls, full_symbol_name, known_namespaces): + chunks = full_symbol_name.split('.') + namespaces, name = chunks[:-1], chunks[-1] + classes = [] + while len(namespaces) > 0 and '.'.join(namespaces) not in known_namespaces: + classes.insert(0, namespaces.pop()) + return cls(tuple(namespaces), tuple(classes), name) + + setattr(SymbolName, "parse", classmethod(parse_symbol_name)) + forbidden_arg_types = ["void*"] @@ -182,6 +200,7 @@ ${variant} } """) + class FormatStrings: string = 's' unsigned_char = 'b' @@ -197,9 +216,9 @@ class FormatStrings: double = 'd' object = 'O' + ArgTypeInfo = namedtuple('ArgTypeInfo', - ['atype', 'format_str', 'default_value', - 'strict_conversion']) + ['atype', 'format_str', 'default_value', 'strict_conversion']) # strict_conversion is False by default ArgTypeInfo.__new__.__defaults__ = (False,) @@ -256,12 +275,16 @@ class ClassProp(object): class ClassInfo(object): def __init__(self, name, decl=None, codegen=None): # Scope name can be a module or other class e.g. cv::SimpleBlobDetector::Params - scope_name, self.original_name = name.rsplit(".", 1) + self.original_scope_name, self.original_name = name.rsplit(".", 1) # In case scope refer the outer class exported with different name if codegen: - scope_name = codegen.get_export_scope_name(scope_name) - self.scope_name = re.sub(r"^cv\.?", "", scope_name) + self.export_scope_name = codegen.get_export_scope_name( + self.original_scope_name + ) + else: + self.export_scope_name = self.original_scope_name + self.export_scope_name = re.sub(r"^cv\.?", "", self.export_scope_name) self.export_name = self.original_name @@ -312,8 +335,8 @@ class ClassInfo(object): @property def wname(self): - if len(self.scope_name) > 0: - return self.scope_name.replace(".", "_") + "_" + self.export_name + if len(self.export_scope_name) > 0: + return self.export_scope_name.replace(".", "_") + "_" + self.export_name return self.export_name @@ -322,16 +345,16 @@ class ClassInfo(object): return self.class_id @property - def full_scope_name(self): - return "cv." + self.scope_name if len(self.scope_name) else "cv" + def full_export_scope_name(self): + return "cv." + self.export_scope_name if len(self.export_scope_name) else "cv" @property def full_export_name(self): - return self.full_scope_name + "." + self.export_name + return self.full_export_scope_name + "." + self.export_name @property def full_original_name(self): - return self.full_scope_name + "." + self.original_name + return self.original_scope_name + "." + self.original_name @property def has_export_alias(self): @@ -415,7 +438,7 @@ class ClassInfo(object): baseptr, constructor_name, # Leading dot is required to provide correct class naming - "." + self.scope_name if len(self.scope_name) > 0 else self.scope_name + "." + self.export_scope_name if len(self.export_scope_name) > 0 else self.export_scope_name ) @@ -577,6 +600,10 @@ class FuncVariant(object): self.args.append(ainfo) self.init_pyproto(namespace, classname, known_classes) + def is_arg_optional(self, py_arg_index): + # type: (FuncVariant, int) -> bool + return py_arg_index >= len(self.py_arglist) - self.py_noptargs + def init_pyproto(self, namespace, classname, known_classes): # string representation of argument list, with '[', ']' symbols denoting optional arguments, e.g. # "src1, src2[, dst[, mask]]" for cv.add @@ -1053,6 +1080,7 @@ class PythonWrapperGenerator(object): self.namespaces = {} self.consts = {} self.enums = {} + self.typing_stubs_generator = TypingStubsGenerator() self.code_include = StringIO() self.code_enums = StringIO() self.code_types = StringIO() @@ -1099,26 +1127,19 @@ class PythonWrapperGenerator(object): return original_scope_name def split_decl_name(self, name): - chunks = name.split('.') - namespace = chunks[:-1] - classes = [] - while namespace and '.'.join(namespace) not in self.parser.namespaces: - classes.insert(0, namespace.pop()) - return namespace, classes, chunks[-1] - + return SymbolName.parse(name, self.parser.namespaces) def add_const(self, name, decl): cname = name.replace('.','::') namespace, classes, name = self.split_decl_name(name) namespace = '.'.join(namespace) - name = '_'.join(classes+[name]) + name = '_'.join(chain(classes, (name, ))) ns = self.namespaces.setdefault(namespace, Namespace()) if name in ns.consts: print("Generator error: constant %s (cname=%s) already exists" \ % (name, cname)) sys.exit(-1) ns.consts[name] = cname - value = decl[1] py_name = '.'.join([namespace, name]) py_signatures = self.py_signatures.setdefault(cname, []) @@ -1126,20 +1147,31 @@ class PythonWrapperGenerator(object): #print(cname + ' => ' + str(py_name) + ' (value=' + value + ')') def add_enum(self, name, decl): + enumeration_name = SymbolName.parse(name, self.parser.namespaces) + is_scoped_enum = decl[0].startswith("enum class") \ + or decl[0].startswith("enum struct") + wname = normalize_class_name(name) if wname.endswith(""): wname = None else: self.enums[wname] = name const_decls = decl[3] - + enum_entries = {} for decl in const_decls: - name = decl[0] - self.add_const(name.replace("const ", "").strip(), decl) + enum_entries[decl[0].split(".")[-1]] = decl[1] + + self.add_const(decl[0].replace("const ", "").strip(), decl) + + # Extra enumerations tracking is required to generate stubs for + # all enumerations, including once, otherwise they + # will be forgiven + self.typing_stubs_generator.add_enum(enumeration_name, is_scoped_enum, + enum_entries) def add_func(self, decl): namespace, classes, barename = self.split_decl_name(decl[0]) - cname = "::".join(namespace+classes+[barename]) + cname = "::".join(chain(namespace, classes, (barename, ))) name = barename classname = '' bareclassname = '' @@ -1166,7 +1198,7 @@ class PythonWrapperGenerator(object): return if isconstructor: - name = "_".join(classes[:-1]+[name]) + name = "_".join(chain(classes[:-1], (name, ))) if is_static: # Add it as a method to the class @@ -1175,7 +1207,7 @@ class PythonWrapperGenerator(object): func.add_variant(decl, self.classes, isphantom) # Add it as global function - g_name = "_".join(classes+[name]) + g_name = "_".join(chain(classes, (name, ))) w_classes = [] for i in range(0, len(classes)): classes_i = classes[:i+1] @@ -1187,10 +1219,15 @@ class PythonWrapperGenerator(object): w_classes.append(w_classname) g_wname = "_".join(w_classes+[name]) func_map = self.namespaces.setdefault(namespace_str, Namespace()).funcs + # Static functions should be called using class names, not like + # module-level functions, so first step is to remove them from + # type hints. + self.typing_stubs_generator.add_ignored_function_name(g_name) # Exports static function with internal name (backward compatibility) func = func_map.setdefault(g_name, FuncInfo("", g_name, cname, isconstructor, namespace_str, False)) func.add_variant(decl, self.classes, isphantom) if g_wname != g_name: # TODO OpenCV 5.0 + self.typing_stubs_generator.add_ignored_function_name(g_wname) wfunc = func_map.setdefault(g_wname, FuncInfo("", g_wname, cname, isconstructor, namespace_str, False)) wfunc.add_variant(decl, self.classes, isphantom) else: @@ -1207,7 +1244,6 @@ class PythonWrapperGenerator(object): if classname and isconstructor: self.classes[classname].constructor = func - def gen_namespace(self, ns_name): ns = self.namespaces[ns_name] wname = normalize_class_name(ns_name) @@ -1260,6 +1296,7 @@ class PythonWrapperGenerator(object): self.clear() self.parser = hdr_parser.CppHeaderParser(generate_umat_decls=True, generate_gpumat_decls=True) + # step 1: scan the headers and build more descriptive maps of classes, consts, functions for hdr in srcfiles: decls = self.parser.parse(hdr) @@ -1355,23 +1392,36 @@ class PythonWrapperGenerator(object): for decl_idx, name, classinfo in classlist1: if classinfo.ismap: continue + def _registerType(classinfo): if classinfo.decl_idx in published_types: #print(classinfo.decl_idx, classinfo.name, ' - already published') - return + # If class already registered it means that there is + # a correponding node in the AST. This check is partically + # useful for base classes. + return self.typing_stubs_generator.find_class_node( + classinfo, self.parser.namespaces + ) published_types.add(classinfo.decl_idx) + # Registering a class means creation of the AST node from the + # given class information + class_node = self.typing_stubs_generator.create_class_node( + classinfo, self.parser.namespaces + ) + if classinfo.base and classinfo.base in self.classes: base_classinfo = self.classes[classinfo.base] - #print(classinfo.decl_idx, classinfo.name, ' - request publishing of base type ', base_classinfo.decl_idx, base_classinfo.name) - _registerType(base_classinfo) + # print(classinfo.decl_idx, classinfo.name, ' - request publishing of base type ', base_classinfo.decl_idx, base_classinfo.name) + base_node = _registerType(base_classinfo) + class_node.add_base(base_node) - #print(classinfo.decl_idx, classinfo.name, ' - published!') + # print(classinfo.decl_idx, classinfo.name, ' - published!') self.code_type_publish.write(classinfo.gen_def(self)) + return class_node _registerType(classinfo) - # step 3: generate the code for all the global functions for ns_name, ns in sorted(self.namespaces.items()): if ns_name.split('.')[0] != 'cv': @@ -1381,6 +1431,10 @@ class PythonWrapperGenerator(object): continue code = func.gen_code(self) self.code_funcs.write(code) + # If function is not ignored - create an AST node for it + if name not in self.typing_stubs_generator.type_hints_ignored_functions: + self.typing_stubs_generator.create_function_node(func) + self.gen_namespace(ns_name) self.code_ns_init.write('CVPY_MODULE("{}", {});\n'.format(ns_name[2:], normalize_class_name(ns_name))) @@ -1396,6 +1450,10 @@ class PythonWrapperGenerator(object): for name, constinfo in constlist: self.gen_const_reg(constinfo) + # All symbols are collected and AST is reconstructed, generating + # typing stubs... + self.typing_stubs_generator.generate(output_path) + # That's it. Now save all the files self.save(output_path, "pyopencv_generated_include.h", self.code_include) self.save(output_path, "pyopencv_generated_funcs.h", self.code_funcs) @@ -1406,6 +1464,7 @@ class PythonWrapperGenerator(object): self.save(output_path, "pyopencv_generated_modules_content.h", self.code_ns_reg) self.save_json(output_path, "pyopencv_signatures.json", self.py_signatures) + if __name__ == "__main__": srcfiles = hdr_parser.opencv_hdr_list dstdir = "/Users/vp/tmp" diff --git a/modules/python/src2/typing_stubs_generation/__init__.py b/modules/python/src2/typing_stubs_generation/__init__.py new file mode 100644 index 0000000000..20ebf5773f --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/__init__.py @@ -0,0 +1,34 @@ +from .nodes import ( + NamespaceNode, + ClassNode, + ClassProperty, + EnumerationNode, + FunctionNode, + ConstantNode, + TypeNode, + OptionalTypeNode, + TupleTypeNode, + AliasTypeNode, + SequenceTypeNode, + AnyTypeNode, + AggregatedTypeNode, +) + +from .types_conversion import ( + replace_template_parameters_with_placeholders, + get_template_instantiation_type, + create_type_node +) + +from .ast_utils import ( + SymbolName, + ScopeNotFoundError, + SymbolNotFoundError, + find_scope, + find_class_node, + create_class_node, + create_function_node, + resolve_enum_scopes +) + +from .generation import generate_typing_stubs diff --git a/modules/python/src2/typing_stubs_generation/ast_utils.py b/modules/python/src2/typing_stubs_generation/ast_utils.py new file mode 100644 index 0000000000..1f802f6da5 --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/ast_utils.py @@ -0,0 +1,369 @@ +from typing import NamedTuple, Sequence, Tuple, Union, List, Dict +import keyword + +from .nodes import (ASTNode, NamespaceNode, ClassNode, FunctionNode, + EnumerationNode, ClassProperty, OptionalTypeNode, + TupleTypeNode) + +from .types_conversion import create_type_node + + +class ScopeNotFoundError(Exception): + pass + + +class SymbolNotFoundError(Exception): + pass + + +class SymbolName(NamedTuple): + namespaces: Tuple[str, ...] + classes: Tuple[str, ...] + name: str + + def __str__(self) -> str: + return '(namespace="{}", classes="{}", name="{}")'.format( + '::'.join(self.namespaces), + '::'.join(self.classes), + self.name + ) + + def __repr__(self) -> str: + return str(self) + + @classmethod + def parse(cls, full_symbol_name: str, + known_namespaces: Sequence[str], + symbol_parts_delimiter: str = '.') -> "SymbolName": + """Performs contextual symbol name parsing into namespaces, classes + and "bare" symbol name. + + Args: + full_symbol_name (str): Input string to parse symbol name from. + known_namespaces (Sequence[str]): Collection of namespace that was + met during C++ headers parsing. + symbol_parts_delimiter (str, optional): Delimiter string used to + split `full_symbol_name` string into chunks. Defaults to '.'. + + Returns: + SymbolName: Parsed symbol name structure. + + >>> SymbolName.parse('cv.ns.Feature', ('cv', 'cv.ns')) + (namespace="cv::ns", classes="", name="Feature") + + >>> SymbolName.parse('cv.ns.Feature', ()) + (namespace="", classes="cv::ns", name="Feature") + + >>> SymbolName.parse('cv.ns.Feature.Params', ('cv', 'cv.ns')) + (namespace="cv::ns", classes="Feature", name="Params") + + >>> SymbolName.parse('cv::ns::Feature::Params::serialize', + ... known_namespaces=('cv', 'cv.ns'), + ... symbol_parts_delimiter='::') + (namespace="cv::ns", classes="Feature::Params", name="serialize") + """ + + chunks = full_symbol_name.split(symbol_parts_delimiter) + namespaces, name = chunks[:-1], chunks[-1] + classes: List[str] = [] + while len(namespaces) > 0 and '.'.join(namespaces) not in known_namespaces: + classes.insert(0, namespaces.pop()) + return SymbolName(tuple(namespaces), tuple(classes), name) + + +def find_scope(root: NamespaceNode, symbol_name: SymbolName, + create_missing_namespaces: bool = True) -> Union[NamespaceNode, ClassNode]: + """Traverses down nodes hierarchy to the direct parent of the node referred + by `symbol_name`. + + Args: + root (NamespaceNode): Root node of the hierarchy. + symbol_name (SymbolName): Full symbol name to find scope for. + create_missing_namespaces (bool, optional): Set to True to create missing + namespaces while traversing the hierarchy. Defaults to True. + + Raises: + ScopeNotFoundError: If direct parent for the node referred by `symbol_name` + can't be found e.g. one of classes doesn't exist. + + Returns: + Union[NamespaceNode, ClassNode]: Direct parent for the node referred by + `symbol_name`. + + >>> root = NamespaceNode('cv') + >>> algorithm_node = root.add_class('Algorithm') + >>> find_scope(root, SymbolName(('cv', ), ('Algorithm',), 'Params')) == algorithm_node + True + + >>> root = NamespaceNode('cv') + >>> scope = find_scope(root, SymbolName(('cv', 'gapi', 'detail'), (), 'function')) + >>> scope.full_export_name + 'cv.gapi.detail' + + >>> root = NamespaceNode('cv') + >>> scope = find_scope(root, SymbolName(('cv', 'gapi'), ('GOpaque',), 'function')) + Traceback (most recent call last): + ... + ast_utils.ScopeNotFoundError: Can't find a scope for 'function', with \ +'(namespace="cv::gapi", classes="GOpaque", name="function")', \ +because 'GOpaque' class is not registered yet + """ + assert isinstance(root, NamespaceNode), \ + 'Wrong hierarchy root type: {}'.format(type(root)) + + assert symbol_name.namespaces[0] == root.name, \ + "Trying to find scope for '{}' with root namespace different from: '{}'".format( + symbol_name, root.name + ) + + scope: Union[NamespaceNode, ClassNode] = root + for namespace in symbol_name.namespaces[1:]: + if namespace not in scope.namespaces: # type: ignore + if not create_missing_namespaces: + raise ScopeNotFoundError( + "Can't find a scope for '{}', with '{}', because namespace" + " '{}' is not created yet and `create_missing_namespaces`" + " flag is set to False".format( + symbol_name.name, symbol_name, namespace + ) + ) + scope = scope.add_namespace(namespace) # type: ignore + else: + scope = scope.namespaces[namespace] # type: ignore + for class_name in symbol_name.classes: + if class_name not in scope.classes: + raise ScopeNotFoundError( + "Can't find a scope for '{}', with '{}', because '{}' " + "class is not registered yet".format( + symbol_name.name, symbol_name, class_name + ) + ) + scope = scope.classes[class_name] + return scope + + +def find_class_node(root: NamespaceNode, full_class_name: str, + namespaces: Sequence[str]) -> ClassNode: + symbol_name = SymbolName.parse(full_class_name, namespaces) + scope = find_scope(root, symbol_name) + if symbol_name.name not in scope.classes: + raise SymbolNotFoundError("Can't find {} in its scope".format(symbol_name)) + return scope.classes[symbol_name.name] + + +def create_function_node_in_scope(scope: Union[NamespaceNode, ClassNode], + func_info) -> FunctionNode: + def prepare_overload_arguments_and_return_type(variant): + arguments = [] # type: list[FunctionNode.Arg] + # Enumerate is requried, because `argno` in `variant.py_arglist` + # refers to position of argument in C++ function interface, + # but `variant.py_noptargs` refers to position in `py_arglist` + for i, (_, argno) in enumerate(variant.py_arglist): + arg_info = variant.args[argno] + type_node = create_type_node(arg_info.tp) + default_value = None + if len(arg_info.defval): + default_value = arg_info.defval + # If argument is optional and can be None - make its type optional + if variant.is_arg_optional(i): + # NOTE: should UMat be always mandatory for better type hints? + # otherwise overload won't be selected e.g. VideoCapture.read() + if arg_info.py_outputarg: + type_node = OptionalTypeNode(type_node) + default_value = "None" + elif arg_info.isbig() and "None" not in type_node.typename: + # but avoid duplication of the optioness + type_node = OptionalTypeNode(type_node) + arguments.append( + FunctionNode.Arg(arg_info.export_name, type_node=type_node, + default_value=default_value) + ) + if func_info.isconstructor: + return arguments, None + + # Function has more than 1 output argument, so its return type is a tuple + if len(variant.py_outlist) > 1: + ret_types = [] + # Actual returned value of the function goes first + if variant.py_outlist[0][1] == -1: + ret_types.append(create_type_node(variant.rettype)) + outlist = variant.py_outlist[1:] + else: + outlist = variant.py_outlist + for _, argno in outlist: + assert argno >= 0, \ + "Logic Error! Outlist contains function return type: {}".format( + outlist + ) + + ret_types.append(create_type_node(variant.args[argno].tp)) + + return arguments, FunctionNode.RetType( + TupleTypeNode("return_type", ret_types) + ) + # Function with 1 output argument in Python + if len(variant.py_outlist) == 1: + # Can be represented as a function with a non-void return type in C++ + if variant.rettype: + return arguments, FunctionNode.RetType( + create_type_node(variant.rettype) + ) + # or a function with void return type and output argument type + # such non-const reference + ret_type = variant.args[variant.py_outlist[0][1]].tp + return arguments, FunctionNode.RetType( + create_type_node(ret_type) + ) + # Function without output types returns None in Python + return arguments, None + + function_node = FunctionNode(func_info.name) + function_node.parent = scope + if func_info.isconstructor: + function_node.export_name = "__init__" + for variant in func_info.variants: + arguments, ret_type = prepare_overload_arguments_and_return_type(variant) + if isinstance(scope, ClassNode): + if func_info.is_static: + if ret_type is not None and ret_type.typename.endswith(scope.name): + function_node.is_classmethod = True + arguments.insert(0, FunctionNode.Arg("cls")) + else: + function_node.is_static = True + else: + arguments.insert(0, FunctionNode.Arg("self")) + function_node.add_overload(arguments, ret_type) + return function_node + + +def create_function_node(root: NamespaceNode, func_info) -> FunctionNode: + func_symbol_name = SymbolName( + func_info.namespace.split(".") if len(func_info.namespace) else (), + func_info.classname.split(".") if len(func_info.classname) else (), + func_info.name + ) + return create_function_node_in_scope(find_scope(root, func_symbol_name), + func_info) + + +def create_class_node_in_scope(scope: Union[NamespaceNode, ClassNode], + symbol_name: SymbolName, + class_info) -> ClassNode: + properties = [] + for property in class_info.props: + export_property_name = property.name + if keyword.iskeyword(export_property_name): + export_property_name += "_" + properties.append( + ClassProperty( + name=export_property_name, + type_node=create_type_node(property.tp), + is_readonly=property.readonly + ) + ) + class_node = scope.add_class(symbol_name.name, + properties=properties) + class_node.export_name = class_info.export_name + if class_info.constructor is not None: + create_function_node_in_scope(class_node, class_info.constructor) + for method in class_info.methods.values(): + create_function_node_in_scope(class_node, method) + return class_node + + +def create_class_node(root: NamespaceNode, class_info, + namespaces: Sequence[str]) -> ClassNode: + symbol_name = SymbolName.parse(class_info.full_original_name, namespaces) + scope = find_scope(root, symbol_name) + return create_class_node_in_scope(scope, symbol_name, class_info) + + +def resolve_enum_scopes(root: NamespaceNode, + enums: Dict[SymbolName, EnumerationNode]): + """Attaches all enumeration nodes to the appropriate classes and modules + + If classes containing enumeration can't be found in the AST - they will + be created and marked as not exportable. This behavior is required to cover + cases, when enumeration is defined in base class, but only its derivatives + are used. Example: + ```cpp + class CV_EXPORTS TermCriteria { + public: + enum Type { /* ... */ }; + // ... + }; + ``` + + Args: + root (NamespaceNode): root of the reconstructed AST + enums (Dict[SymbolName, EnumerationNode]): Mapping between enumerations + symbol names and corresponding nodes without parents. + """ + + for symbol_name, enum_node in enums.items(): + if symbol_name.classes: + try: + scope = find_scope(root, symbol_name) + except ScopeNotFoundError: + # Scope can't be found if enumeration is a part of class + # that is not exported. + # Create class node, but mark it as not exported + for i, class_name in enumerate(symbol_name.classes): + scope = find_scope(root, + SymbolName(symbol_name.namespaces, + classes=symbol_name.classes[:i], + name=class_name)) + if class_name in scope.classes: + continue + class_node = scope.add_class(class_name) + class_node.is_exported = False + scope = find_scope(root, symbol_name) + else: + scope = find_scope(root, symbol_name) + enum_node.parent = scope + + +def get_enclosing_namespace(node: ASTNode) -> NamespaceNode: + """Traverses up nodes hierarchy to find closest enclosing namespace of the + passed node + + Args: + node (ASTNode): Node to find a namespace for. + + Returns: + NamespaceNode: Closest enclosing namespace of the provided node. + + Raises: + AssertionError: if nodes hierarchy missing a namespace node. + + >>> root = NamespaceNode('cv') + >>> feature_class = root.add_class("Feature") + >>> get_enclosing_namespace(feature_class) == root + True + + >>> root = NamespaceNode('cv') + >>> feature_class = root.add_class("Feature") + >>> feature_params_class = feature_class.add_class("Params") + >>> serialize_params_func = feature_params_class.add_function("serialize") + >>> get_enclosing_namespace(serialize_params_func) == root + True + + >>> root = NamespaceNode('cv') + >>> detail_ns = root.add_namespace('detail') + >>> flags_enum = detail_ns.add_enumeration('Flags') + >>> get_enclosing_namespace(flags_enum) == detail_ns + True + """ + parent_node = node.parent + while not isinstance(parent_node, NamespaceNode): + assert parent_node is not None, \ + "Can't find enclosing namespace for '{}' known as: '{}'".format( + node.full_export_name, node.native_name + ) + parent_node = parent_node.parent + return parent_node + + +if __name__ == '__main__': + import doctest + doctest.testmod() diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py new file mode 100644 index 0000000000..018c55414a --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -0,0 +1,671 @@ +__all__ = ("generate_typing_stubs", ) + +from io import StringIO +from pathlib import Path +from typing import (Generator, Type, Callable, NamedTuple, Union, Set, Dict, + Collection) +import warnings + +from .ast_utils import get_enclosing_namespace + +from .predefined_types import PREDEFINED_TYPES + +from .nodes import (ASTNode, NamespaceNode, ClassNode, FunctionNode, + EnumerationNode, ConstantNode) + +from .nodes.type_node import (TypeNode, AliasTypeNode, AliasRefTypeNode, + AggregatedTypeNode) + + +def generate_typing_stubs(root: NamespaceNode, output_path: Path): + """Generates typing stubs for the AST with root `root` and outputs + created files tree to directory pointed by `output_path`. + + Stubs generation consist from 4 steps: + 1. Reconstruction of AST tree for header parser output. + 2. "Lazy" AST nodes resolution (type nodes used as function arguments + and return types). Resolution procedure attaches every "lazy" + AST node to the corresponding node in the AST created during step 1. + 3. Generation of the typing module content. Typing module doesn't exist + in library code, but is essential place to define aliases widely used + in stub files. + 4. Generation of typing stubs from the reconstructed AST. + Every namespace corresponds to a Python module with the same name. + Generation procedure is recursive repetition of the following steps + for each namespace (module): + - Collect and write required imports for the module + - Write all module constants stubs + - Write all module enumerations stubs + - Write all module classes stubs, preserving correct declaration + order, when base classes go before their derivatives. + - Write all module functions stubs + - Repeat steps above for nested namespaces + + Args: + root (NamespaceNode): Root namespace node of the library AST. + output_path (Path): Path to output directory. + """ + # Most of the time type nodes miss their full name (especially function + # arguments and return types), so resolution should start from the narrowest + # scope and gradually expanded. + # Example: + # ```cpp + # namespace cv { + # enum AlgorithmType { + # // ... + # }; + # namespace detail { + # struct Algorithm { + # static Ptr create(AlgorithmType alg_type); + # }; + # } // namespace detail + # } // namespace cv + # ``` + # To resolve `alg_type` argument of function `create` having `AlgorithmType` + # type from above example the following steps are done: + # 1. Try to resolve against `cv::detail::Algorithm` - fail + # 2. Try to resolve against `cv::detail` - fail + # 3. Try to resolve against `cv` - success + # The whole process should fail !only! when all possible scopes are + # checked and at least 1 node is still unresolved. + root.resolve_type_nodes() + _generate_typing_module(root, output_path) + _generate_typing_stubs(root, output_path) + + +def _generate_typing_stubs(root: NamespaceNode, output_path: Path): + output_path = Path(output_path) / root.export_name + output_path.mkdir(parents=True, exist_ok=True) + + # Collect all imports required for module items declaration + required_imports = _collect_required_imports(root) + + output_stream = StringIO() + + # Write required imports at the top of file + _write_required_imports(required_imports, output_stream) + + # Write constants section, because constants don't impose any dependencies + _generate_section_stub(StubSection("# Constants", ConstantNode), root, + output_stream, 0) + # NOTE: Enumerations require special handling, because all enumeration + # constants are exposed as module attributes + has_enums = _generate_section_stub(StubSection("# Enumerations", EnumerationNode), + root, output_stream, 0) + # Collect all enums from class level and export them to module level + for class_node in root.classes.values(): + if _generate_enums_from_classes_tree(class_node, output_stream, indent=0): + has_enums = True + # 2 empty lines between enum and classes definitions + if has_enums: + output_stream.write("\n") + + # Write the rest of module content - classes and functions + for section in STUB_SECTIONS: + _generate_section_stub(section, root, output_stream, 0) + # Dump content to the output file + (output_path / "__init__.pyi").write_text(output_stream.getvalue()) + # Process nested namespaces + for ns in root.namespaces.values(): + _generate_typing_stubs(ns, output_path) + + +class StubSection(NamedTuple): + name: str + node_type: Type[ASTNode] + + +STUB_SECTIONS = ( + StubSection("# Constants", ConstantNode), + # StubSection("# Enumerations", EnumerationNode), # Skipped for now (special rules) + StubSection("# Classes", ClassNode), + StubSection("# Functions", FunctionNode) +) + + +def _generate_section_stub(section: StubSection, node: ASTNode, + output_stream: StringIO, indent: int) -> bool: + """Generates stub for a single type of children nodes of the provided node. + + Args: + section (StubSection): section identifier that carries section name and + type its nodes. + node (ASTNode): root node with children nodes used for + output_stream (StringIO): Output stream for all nodes stubs related to + the given section. + indent (int): Indent used for each line written to `output_stream`. + + Returns: + bool: `True` if section has a content, `False` otherwise. + """ + if section.node_type not in node._children: + return False + + children = node._children[section.node_type] + if len(children) == 0: + return False + + output_stream.write(" " * indent) + output_stream.write(section.name) + output_stream.write("\n") + stub_generator = NODE_TYPE_TO_STUB_GENERATOR[section.node_type] + children = filter(lambda c: c.is_exported, children.values()) # type: ignore + if hasattr(section.node_type, "weight"): + children = sorted(children, key=lambda child: getattr(child, "weight")) # type: ignore + for child in children: + stub_generator(child, output_stream, indent) # type: ignore + output_stream.write("\n") + return True + + +def _generate_class_stub(class_node: ClassNode, output_stream: StringIO, + indent: int = 0) -> None: + """Generates stub for the provided class node. + + Rules: + - Read/write properties are converted to object attributes. + - Readonly properties are converted to functions decorated with `@property`. + - When return type of static functions matches class name - these functions + are treated as factory functions and annotated with `@classmethod`. + - In contrast to implicit `this` argument in C++ methods, in Python all + "normal" methods have explicit `self` as their first argument. + - Body of empty classes is replaced with `...` + + Example: + ```cpp + struct Object : public BaseObject { + struct InnerObject { + int param; + bool param2; + + float readonlyParam(); + }; + + Object(int param, bool param2 = false); + + Object(InnerObject obj); + + static Object create(); + + }; + ``` + becomes + ```python + class Object(BaseObject): + class InnerObject: + param: int + param2: bool + + @property + def readonlyParam() -> float: ... + + @typing.override + def __init__(self, param: int, param2: bool = ...) -> None: ... + + @typing.override + def __init__(self, obj: "Object.InnerObject") -> None: ... + + @classmethod + def create(cls) -> Object: ... + ``` + + Args: + class_node (ClassNode): Class node to generate stub entry for. + output_stream (StringIO): Output stream for class stub. + indent (int, optional): Indent used for each line written to + `output_stream`. Defaults to 0. + """ + + class_module = get_enclosing_namespace(class_node) + class_module_name = class_module.full_export_name + + if len(class_node.bases) > 0: + bases = [] + for base in class_node.bases: + base_module = get_enclosing_namespace(base) # type: ignore + if base_module != class_module: + bases.append(base.full_export_name) + else: + bases.append(base.export_name) + + inheritance_str = "({})".format( + ', '.join(bases) + ) + else: + inheritance_str = "" + + output_stream.write( + "{indent}class {name}{bases}:\n".format( + indent=" " * indent, + name=class_node.export_name, + bases=inheritance_str + ) + ) + has_content = len(class_node.properties) > 0 + + # Processing class properties + for property in class_node.properties: + if property.is_readonly: + template = "{indent}@property\n{indent}def {name}(self) -> {type}: ...\n" + else: + template = "{indent}{name}: {type}\n" + + output_stream.write( + template.format(indent=" " * (indent + 4), + name=property.name, + type=property.relative_typename(class_module_name)) + ) + if len(class_node.properties) > 0: + output_stream.write("\n") + + for section in STUB_SECTIONS: + if _generate_section_stub(section, class_node, + output_stream, indent + 4): + has_content = True + if not has_content: + output_stream.write(" " * (indent + 4)) + output_stream.write("...\n\n") + + +def _generate_constant_stub(constant_node: ConstantNode, + output_stream: StringIO, indent: int = 0, + extra_export_prefix: str = "") -> None: + """Generates stub for the provided constant node. + + Args: + constant_node (ConstantNode): Constant node to generate stub entry for. + output_stream (StringIO): Output stream for constant stub. + indent (int, optional): Indent used for each line written to + `output_stream`. Defaults to 0. + extra_export_prefix (str, optional) Extra prefix added to the export + constant name. Defaults to empty string. + """ + + output_stream.write( + "{indent}{prefix}{name}: {value_type}\n".format( + prefix=extra_export_prefix, + name=constant_node.export_name, + value_type=constant_node.value_type, + indent=" " * indent + ) + ) + + +def _generate_enumeration_stub(enumeration_node: EnumerationNode, + output_stream: StringIO, indent: int = 0, + extra_export_prefix: str = "") -> None: + """Generates stub for the provided enumeration node. In contrast to the + Python `enum.Enum` class, C++ enumerations are exported as module-level + (or class-level) constants. + + Example: + ```cpp + enum Flags { + Flag1 = 0, + Flag2 = 1, + Flag3 + }; + ``` + becomes + ```python + Flag1: int + Flag2: int + Flag3: int + Flags = int # One of [Flag1, Flag2, Flag3] + ``` + + Unnamed enumerations don't export their names to Python: + ```cpp + enum { + Flag1 = 0, + Flag2 = 1 + }; + ``` + becomes + ```python + Flag1: int + Flag2: int + ``` + + Scoped enumeration adds its name before each item name: + ```cpp + enum struct ScopedEnum { + Flag1, + Flag2 + }; + ``` + becomes + ```python + ScopedEnum_Flag1: int + ScopedEnum_Flag2: int + ScopedEnum = int # One of [ScopedEnum_Flag1, ScopedEnum_Flag2] + ``` + + Args: + enumeration_node (EnumerationNode): Enumeration node to generate stub entry for. + output_stream (StringIO): Output stream for enumeration stub. + indent (int, optional): Indent used for each line written to `output_stream`. + Defaults to 0. + extra_export_prefix (str, optional) Extra prefix added to the export + enumeration name. Defaults to empty string. + """ + + entries_extra_prefix = extra_export_prefix + if enumeration_node.is_scoped: + entries_extra_prefix += enumeration_node.export_name + "_" + for entry in enumeration_node.constants.values(): + _generate_constant_stub(entry, output_stream, indent, entries_extra_prefix) + # Unnamed enumerations are skipped as definition + if enumeration_node.export_name.endswith(""): + output_stream.write("\n") + return + output_stream.write( + "{indent}{export_prefix}{name} = int # One of [{entries}]\n\n".format( + export_prefix=extra_export_prefix, + name=enumeration_node.export_name, + entries=", ".join(entry.export_name + for entry in enumeration_node.constants.values()), + indent=" " * indent + ) + ) + + +def _generate_function_stub(function_node: FunctionNode, + output_stream: StringIO, indent: int = 0) -> None: + """Generates stub entry for the provided function node. Function node can + refer free function or class method. + + Args: + function_node (FunctionNode): Function node to generate stub entry for. + output_stream (StringIO): Output stream for function stub. + indent (int, optional): Indent used for each line written to + `output_stream`. Defaults to 0. + """ + + # Function is a stub without any arguments information + if not function_node.overloads: + warnings.warn( + 'Function node "{}" exported as "{}" has no overloads'.format( + function_node.full_name, function_node.full_export_name + ) + ) + return + + decorators = [] + if function_node.is_classmethod: + decorators.append(" " * indent + "@classmethod") + elif function_node.is_static: + decorators.append(" " * indent + "@staticmethod") + if len(function_node.overloads) > 1: + decorators.append(" " * indent + "@typing.overload") + + function_module = get_enclosing_namespace(function_node) + function_module_name = function_module.full_export_name + + for overload in function_node.overloads: + # Annotate every function argument + annotated_args = [] + for arg in overload.arguments: + annotated_arg = arg.name + typename = arg.relative_typename(function_module_name) + if typename is not None: + annotated_arg += ": " + typename + if arg.default_value is not None: + annotated_arg += " = ..." + annotated_args.append(annotated_arg) + + # And convert return type to the actual type + if overload.return_type is not None: + ret_type = overload.return_type.relative_typename(function_module_name) + else: + ret_type = "None" + + output_stream.write( + "{decorators}" + "{indent}def {name}({args}) -> {ret_type}: ...\n".format( + decorators="\n".join(decorators) + + "\n" if len(decorators) > 0 else "", + name=function_node.export_name, + args=", ".join(annotated_args), + ret_type=ret_type, + indent=" " * indent + ) + ) + output_stream.write("\n") + + +def _generate_enums_from_classes_tree(class_node: ClassNode, + output_stream: StringIO, + indent: int = 0, + class_name_prefix: str = "") -> bool: + """Recursively generates class-level enumerations on the module level + starting from the `class_node`. + + NOTE: This function is required, because all enumerations are exported as + module-level constants. + + Example: + ```cpp + namespace cv { + struct TermCriteria { + enum Type { + COUNT = 1, + MAX_ITER = COUNT, + EPS = 2 + }; + }; + } // namespace cv + ``` + is exported to `__init__.pyi` of `cv` module as as + ```python + TermCriteria_COUNT: int + TermCriteria_MAX_ITER: int + TermCriteria_EPS: int + TermCriteria_Type = int # One of [COUNT, MAX_ITER, EPS] + ``` + + Args: + class_node (ClassNode): Class node to generate enumerations stubs for. + output_stream (StringIO): Output stream for enumerations stub. + indent (int, optional): Indent used for each line written to + `output_stream`. Defaults to 0. + class_name_prefix (str, optional): Prefix used for enumerations and + constants names. Defaults to "". + + Returns: + bool: `True` if classes tree declares at least 1 enum, `False` otherwise. + """ + + class_name_prefix = class_node.export_name + "_" + class_name_prefix + has_content = len(class_node.enumerations) > 0 + for enum_node in class_node.enumerations.values(): + _generate_enumeration_stub(enum_node, output_stream, indent, + class_name_prefix) + for cls in class_node.classes.values(): + if _generate_enums_from_classes_tree(cls, output_stream, indent, + class_name_prefix): + has_content = True + return has_content + + +def check_overload_presence(node: Union[NamespaceNode, ClassNode]) -> bool: + """Checks that node has at least 1 function with overload. + + Args: + node (Union[NamespaceNode, ClassNode]): Node to check for overload + presence. + + Returns: + bool: True if input node has at least 1 function with overload, False + otherwise. + """ + for func_node in node.functions.values(): + if len(func_node.overloads): + return True + return False + + +def _for_each_class(node: Union[NamespaceNode, ClassNode]) \ + -> Generator[ClassNode, None, None]: + for cls in node.classes.values(): + yield cls + if len(cls.classes): + yield from _for_each_class(cls) + + +def _for_each_function(node: Union[NamespaceNode, ClassNode]) \ + -> Generator[FunctionNode, None, None]: + for func in node.functions.values(): + yield func + for cls in node.classes.values(): + yield from _for_each_function(cls) + + +def _for_each_function_overload(node: Union[NamespaceNode, ClassNode]) \ + -> Generator[FunctionNode.Overload, None, None]: + for func in _for_each_function(node): + for overload in func.overloads: + yield overload + + +def _collect_required_imports(root: NamespaceNode) -> Set[str]: + """Collects all imports required for classes and functions typing stubs + declarations. + + Args: + root (NamespaceNode): Namespace node to collect imports for + + Returns: + Set[str]: Collection of unique `import smth` statements required for + classes and function declarations of `root` node. + """ + + def _add_required_usage_imports(type_node: TypeNode, imports: Set[str]): + for required_import in type_node.required_usage_imports: + imports.add(required_import) + + required_imports: Set[str] = set() + # Check if typing module is required due to @overload decorator usage + # Looking for module-level function with at least 1 overload + has_overload = check_overload_presence(root) + # if there is no module-level functions with overload, check its presence + # during class traversing, including their inner-classes + for cls in _for_each_class(root): + if not has_overload and check_overload_presence(cls): + has_overload = True + required_imports.add("import typing") + # Add required imports for class properties + for prop in cls.properties: + _add_required_usage_imports(prop.type_node, required_imports) + # Add required imports for class bases + for base in cls.bases: + base_namespace = get_enclosing_namespace(base) # type: ignore + if base_namespace != root: + required_imports.add( + "import " + base_namespace.full_export_name + ) + + if has_overload: + required_imports.add("import typing") + # Importing modules required to resolve functions arguments + for overload in _for_each_function_overload(root): + for arg in filter(lambda a: a.type_node is not None, overload.arguments): + _add_required_usage_imports(arg.type_node, required_imports) # type: ignore + if overload.return_type is not None: + _add_required_usage_imports(overload.return_type.type_node, + required_imports) + + root_import = "import " + root.full_export_name + if root_import in required_imports: + required_imports.remove(root_import) + + return required_imports + + +def _write_required_imports(required_imports: Collection[str], + output_stream: StringIO) -> None: + """Writes all entries of `required_imports` to the `output_stream`. + + Args: + required_imports (Collection[str]): Imports to write into the output + stream. + output_stream (StringIO): Output stream for import statements. + """ + + for required_import in sorted(required_imports): + output_stream.write(required_import) + output_stream.write("\n") + if len(required_imports): + output_stream.write("\n\n") + + +def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None: + """Generates stub file for typings module. + Actual module doesn't exist, but it is an appropriate place to define + all widely-used aliases. + + Args: + root (NamespaceNode): AST root node used for type nodes resolution. + output_path (Path): Path to typing module directory, where __init__.pyi + will be written. + """ + def register_alias_links_from_aggregated_type(type_node: TypeNode) -> None: + assert isinstance(type_node, AggregatedTypeNode), \ + "Provided type node '{}' is not an aggregated type".format( + type_node.ctype_name + ) + + for item in filter(lambda i: isinstance(i, AliasRefTypeNode), type_node): + register_alias(PREDEFINED_TYPES[item.ctype_name]) # type: ignore + + def register_alias(alias_node: AliasTypeNode) -> None: + typename = alias_node.typename + # Check if alias is already registered + if typename in aliases: + return + if isinstance(alias_node.value, AggregatedTypeNode): + # Check if collection contains a link to another alias + register_alias_links_from_aggregated_type(alias_node.value) + + # Strip module prefix from aliased types + aliases[typename] = alias_node.value.full_typename.replace( + root.export_name + ".typing.", "" + ) + if alias_node.comment is not None: + aliases[typename] += " # " + alias_node.comment + for required_import in alias_node.required_definition_imports: + required_imports.add(required_import) + + output_path = Path(output_path) / root.export_name / "typing" + output_path.mkdir(parents=True, exist_ok=True) + + required_imports: Set[str] = set() + aliases: Dict[str, str] = {} + + # Resolve each node and register aliases + for node in PREDEFINED_TYPES.values(): + node.resolve(root) + if isinstance(node, AliasTypeNode): + register_alias(node) + + output_stream = StringIO() + _write_required_imports(required_imports, output_stream) + + for alias_name, alias_type in aliases.items(): + output_stream.write(alias_name) + output_stream.write(" = ") + output_stream.write(alias_type) + output_stream.write("\n") + + (output_path / "__init__.pyi").write_text(output_stream.getvalue()) + + +StubGenerator = Callable[[ASTNode, StringIO, int], None] + + +NODE_TYPE_TO_STUB_GENERATOR = { + ClassNode: _generate_class_stub, + ConstantNode: _generate_constant_stub, + EnumerationNode: _generate_enumeration_stub, + FunctionNode: _generate_function_stub +} diff --git a/modules/python/src2/typing_stubs_generation/nodes/__init__.py b/modules/python/src2/typing_stubs_generation/nodes/__init__.py new file mode 100644 index 0000000000..fafcd9bc2b --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/nodes/__init__.py @@ -0,0 +1,11 @@ +from .node import ASTNode +from .namespace_node import NamespaceNode +from .class_node import ClassNode, ClassProperty +from .function_node import FunctionNode +from .enumeration_node import EnumerationNode +from .constant_node import ConstantNode +from .type_node import ( + TypeNode, OptionalTypeNode, UnionTypeNode, NoneTypeNode, TupleTypeNode, + ASTNodeTypeNode, AliasTypeNode, SequenceTypeNode, AnyTypeNode, + AggregatedTypeNode, NDArrayTypeNode, AliasRefTypeNode, +) diff --git a/modules/python/src2/typing_stubs_generation/nodes/class_node.py b/modules/python/src2/typing_stubs_generation/nodes/class_node.py new file mode 100644 index 0000000000..b3d394786f --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/nodes/class_node.py @@ -0,0 +1,181 @@ +from typing import Type, Sequence, NamedTuple, Optional, Tuple, Dict +import itertools + +import weakref + +from .node import ASTNode, ASTNodeType + +from .function_node import FunctionNode +from .enumeration_node import EnumerationNode +from .constant_node import ConstantNode + +from .type_node import TypeNode, TypeResolutionError + + +class ClassProperty(NamedTuple): + name: str + type_node: TypeNode + is_readonly: bool + + @property + def typename(self) -> str: + return self.type_node.full_typename + + def resolve_type_nodes(self, root: ASTNode) -> None: + try: + self.type_node.resolve(root) + except TypeResolutionError as e: + raise TypeResolutionError( + 'Failed to resolve "{}" property'.format(self.name) + ) from e + + def relative_typename(self, full_node_name: str) -> str: + """Typename relative to the passed AST node name. + + Args: + full_node_name (str): Full export name of the AST node + + Returns: + str: typename relative to the passed AST node name + """ + return self.type_node.relative_typename(full_node_name) + + +class ClassNode(ASTNode): + """Represents a C++ class that is also a class in Python. + + ClassNode can have functions (methods), enumerations, constants and other + classes as its children nodes. + + Class properties are not treated as a part of AST for simplicity and have + extra handling if required. + """ + def __init__(self, name: str, parent: Optional[ASTNode] = None, + export_name: Optional[str] = None, + bases: Sequence["weakref.ProxyType[ClassNode]"] = (), + properties: Sequence[ClassProperty] = ()) -> None: + super().__init__(name, parent, export_name) + self.bases = list(bases) + self.properties = properties + + @property + def weight(self) -> int: + return 1 + sum(base.weight for base in self.bases) + + @property + def children_types(self) -> Tuple[Type[ASTNode], ...]: + return (ClassNode, FunctionNode, EnumerationNode, ConstantNode) + + @property + def node_type(self) -> ASTNodeType: + return ASTNodeType.Class + + @property + def classes(self) -> Dict[str, "ClassNode"]: + return self._children[ClassNode] + + @property + def functions(self) -> Dict[str, FunctionNode]: + return self._children[FunctionNode] + + @property + def enumerations(self) -> Dict[str, EnumerationNode]: + return self._children[EnumerationNode] + + @property + def constants(self) -> Dict[str, ConstantNode]: + return self._children[ConstantNode] + + def add_class(self, name: str, + bases: Sequence["weakref.ProxyType[ClassNode]"] = (), + properties: Sequence[ClassProperty] = ()) -> "ClassNode": + return self._add_child(ClassNode, name, bases=bases, + properties=properties) + + def add_function(self, name: str, arguments: Sequence[FunctionNode.Arg] = (), + return_type: Optional[FunctionNode.RetType] = None, + is_static: bool = False) -> FunctionNode: + """Adds function as a child node of a class. + + Function is classified in 3 categories: + 1. Instance method. + If function is an instance method then `self` argument is + inserted at the beginning of its arguments list. + + 2. Class method (or factory method) + If `is_static` flag is `True` and typename of the function + return type matches name of the class then function is treated + as class method. + + If function is a class method then `cls` argument is inserted + at the beginning of its arguments list. + + 3. Static method + + Args: + name (str): Name of the function. + arguments (Sequence[FunctionNode.Arg], optional): Function arguments. + Defaults to (). + return_type (Optional[FunctionNode.RetType], optional): Function + return type. Defaults to None. + is_static (bool, optional): Flag whenever function is static or not. + Defaults to False. + + Returns: + FunctionNode: created function node. + """ + + arguments = list(arguments) + if return_type is not None: + is_classmethod = return_type.typename == self.name + else: + is_classmethod = False + if not is_static: + arguments.insert(0, FunctionNode.Arg("self")) + elif is_classmethod: + is_static = False + arguments.insert(0, FunctionNode.Arg("cls")) + return self._add_child(FunctionNode, name, arguments=arguments, + return_type=return_type, is_static=is_static, + is_classmethod=is_classmethod) + + def add_enumeration(self, name: str) -> EnumerationNode: + return self._add_child(EnumerationNode, name) + + def add_constant(self, name: str, value: str) -> ConstantNode: + return self._add_child(ConstantNode, name, value=value) + + def add_base(self, base_class_node: "ClassNode") -> None: + self.bases.append(weakref.proxy(base_class_node)) + + def resolve_type_nodes(self, root: ASTNode) -> None: + """Resolves type nodes for all inner-classes, methods and properties + in 2 steps: + 1. Resolve against `self` as a tree root + 2. Resolve against `root` as a tree root + Type resolution errors are postponed until all children nodes are + examined. + + Args: + root (Optional[ASTNode], optional): Root of the AST sub-tree. + Defaults to None. + """ + + errors = [] + for child in itertools.chain(self.properties, + self.functions.values(), + self.classes.values()): + try: + try: + # Give priority to narrowest scope (class-level scope in this case) + child.resolve_type_nodes(self) # type: ignore + except TypeResolutionError: + child.resolve_type_nodes(root) # type: ignore + except TypeResolutionError as e: + errors.append(str(e)) + if len(errors) > 0: + raise TypeResolutionError( + 'Failed to resolve "{}" class against "{}". Errors: {}'.format( + self.full_export_name, root.full_export_name, errors + ) + ) diff --git a/modules/python/src2/typing_stubs_generation/nodes/constant_node.py b/modules/python/src2/typing_stubs_generation/nodes/constant_node.py new file mode 100644 index 0000000000..63abd8bfb4 --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/nodes/constant_node.py @@ -0,0 +1,30 @@ +from typing import Type, Optional, Tuple + +from .node import ASTNode, ASTNodeType + + +class ConstantNode(ASTNode): + """Represents C++ constant that is also a constant in Python. + """ + def __init__(self, name: str, value: str, + parent: Optional[ASTNode] = None, + export_name: Optional[str] = None) -> None: + super().__init__(name, parent, export_name) + self.value = value + + @property + def children_types(self) -> Tuple[Type[ASTNode], ...]: + return () + + @property + def node_type(self) -> ASTNodeType: + return ASTNodeType.Constant + + @property + def value_type(self) -> str: + return 'int' + + def __str__(self) -> str: + return "Constant('{}' exported as '{}': {})".format( + self.name, self.export_name, self.value + ) diff --git a/modules/python/src2/typing_stubs_generation/nodes/enumeration_node.py b/modules/python/src2/typing_stubs_generation/nodes/enumeration_node.py new file mode 100644 index 0000000000..249f29db1c --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/nodes/enumeration_node.py @@ -0,0 +1,33 @@ +from typing import Type, Tuple, Optional, Dict + +from .node import ASTNode, ASTNodeType + +from .constant_node import ConstantNode + + +class EnumerationNode(ASTNode): + """Represents C++ enumeration that treated as named set of constants in + Python. + + EnumerationNode can have only constants as its children nodes. + """ + def __init__(self, name: str, is_scoped: bool = False, + parent: Optional[ASTNode] = None, + export_name: Optional[str] = None) -> None: + super().__init__(name, parent, export_name) + self.is_scoped = is_scoped + + @property + def children_types(self) -> Tuple[Type[ASTNode], ...]: + return (ConstantNode, ) + + @property + def node_type(self) -> ASTNodeType: + return ASTNodeType.Enumeration + + @property + def constants(self) -> Dict[str, ConstantNode]: + return self._children[ConstantNode] + + def add_constant(self, name: str, value: str) -> ConstantNode: + return self._add_child(ConstantNode, name, value=value) diff --git a/modules/python/src2/typing_stubs_generation/nodes/function_node.py b/modules/python/src2/typing_stubs_generation/nodes/function_node.py new file mode 100644 index 0000000000..a4e4f56561 --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/nodes/function_node.py @@ -0,0 +1,122 @@ +from typing import NamedTuple, Sequence, Type, Optional, Tuple, List + +from .node import ASTNode, ASTNodeType +from .type_node import TypeNode, NoneTypeNode, TypeResolutionError + + +class FunctionNode(ASTNode): + """Represents a function (or class method) in both C++ and Python. + + This class defines an overload set rather then function itself, because + function without overloads is represented as FunctionNode with 1 overload. + """ + class Arg(NamedTuple): + name: str + type_node: Optional[TypeNode] = None + default_value: Optional[str] = None + + @property + def typename(self) -> Optional[str]: + return getattr(self.type_node, "full_typename", None) + + def relative_typename(self, root: str) -> Optional[str]: + if self.type_node is not None: + return self.type_node.relative_typename(root) + return None + + class RetType(NamedTuple): + type_node: TypeNode = NoneTypeNode("void") + + @property + def typename(self) -> str: + return self.type_node.full_typename + + def relative_typename(self, root: str) -> Optional[str]: + return self.type_node.relative_typename(root) + + class Overload(NamedTuple): + arguments: Sequence["FunctionNode.Arg"] = () + return_type: Optional["FunctionNode.RetType"] = None + + def __init__(self, name: str, + arguments: Optional[Sequence["FunctionNode.Arg"]] = None, + return_type: Optional["FunctionNode.RetType"] = None, + is_static: bool = False, + is_classmethod: bool = False, + parent: Optional[ASTNode] = None, + export_name: Optional[str] = None) -> None: + """Function node initializer + + Args: + name (str): Name of the function overload set + arguments (Optional[Sequence[FunctionNode.Arg]], optional): Function + arguments. If this argument is None, then no overloads are + added and node should be treated like a "function stub" rather + than function. This might be helpful if there is a knowledge + that function with the defined name exists, but information + about its interface is not available at that moment. + Defaults to None. + return_type (Optional[FunctionNode.RetType], optional): Function + return type. Defaults to None. + is_static (bool, optional): Flag pointing that function is + a static method of some class. Defaults to False. + is_classmethod (bool, optional): Flag pointing that function is + a class method of some class. Defaults to False. + parent (Optional[ASTNode], optional): Parent ASTNode of the function. + Can be class or namespace. Defaults to None. + export_name (Optional[str], optional): Export name of the function. + Defaults to None. + """ + + super().__init__(name, parent, export_name) + self.overloads: List[FunctionNode.Overload] = [] + self.is_static = is_static + self.is_classmethod = is_classmethod + if arguments is not None: + self.add_overload(arguments, return_type) + + @property + def node_type(self) -> ASTNodeType: + return ASTNodeType.Function + + @property + def children_types(self) -> Tuple[Type[ASTNode], ...]: + return () + + def add_overload(self, arguments: Sequence["FunctionNode.Arg"] = (), + return_type: Optional["FunctionNode.RetType"] = None): + self.overloads.append(FunctionNode.Overload(arguments, return_type)) + + def resolve_type_nodes(self, root: ASTNode): + """Resolves type nodes in all overloads against `root` + + Type resolution errors are postponed until all type nodes are examined. + + Args: + root (ASTNode): Root of AST sub-tree used for type nodes resolution. + """ + def has_unresolved_type_node(item) -> bool: + return item.type_node is not None and not item.type_node.is_resolved + + errors = [] + for overload in self.overloads: + for arg in filter(has_unresolved_type_node, overload.arguments): + try: + arg.type_node.resolve(root) # type: ignore + except TypeResolutionError as e: + errors.append( + 'Failed to resolve "{}" argument: {}'.format(arg.name, e) + ) + if overload.return_type is not None and \ + has_unresolved_type_node(overload.return_type): + try: + overload.return_type.type_node.resolve(root) + except TypeResolutionError as e: + errors.append('Failed to resolve return type: {}'.format(e)) + if len(errors) > 0: + raise TypeResolutionError( + 'Failed to resolve "{}" function against "{}". Errors: {}'.format( + self.full_export_name, root.full_export_name, + ", ".join("[{}]: {}".format(i, e) for i, e in enumerate(errors)) + ) + ) diff --git a/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py b/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py new file mode 100644 index 0000000000..2206f0e586 --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py @@ -0,0 +1,103 @@ +from typing import Type, Iterable, Sequence, Tuple, Optional, Dict +import itertools +import weakref + +from .node import ASTNode, ASTNodeType + +from .class_node import ClassNode, ClassProperty +from .function_node import FunctionNode +from .enumeration_node import EnumerationNode +from .constant_node import ConstantNode + +from .type_node import TypeResolutionError + + +class NamespaceNode(ASTNode): + """Represents C++ namespace that treated as module in Python. + + NamespaceNode can have other namespaces, classes, functions, enumerations + and global constants as its children nodes. + """ + @property + def node_type(self) -> ASTNodeType: + return ASTNodeType.Namespace + + @property + def children_types(self) -> Tuple[Type[ASTNode], ...]: + return (NamespaceNode, ClassNode, FunctionNode, + EnumerationNode, ConstantNode) + + @property + def namespaces(self) -> Dict[str, "NamespaceNode"]: + return self._children[NamespaceNode] + + @property + def classes(self) -> Dict[str, ClassNode]: + return self._children[ClassNode] + + @property + def functions(self) -> Dict[str, FunctionNode]: + return self._children[FunctionNode] + + @property + def enumerations(self) -> Dict[str, EnumerationNode]: + return self._children[EnumerationNode] + + @property + def constants(self) -> Dict[str, ConstantNode]: + return self._children[ConstantNode] + + def add_namespace(self, name: str) -> "NamespaceNode": + return self._add_child(NamespaceNode, name) + + def add_class(self, name: str, + bases: Sequence["weakref.ProxyType[ClassNode]"] = (), + properties: Sequence[ClassProperty] = ()) -> "ClassNode": + return self._add_child(ClassNode, name, bases=bases, + properties=properties) + + def add_function(self, name: str, arguments: Sequence[FunctionNode.Arg] = (), + return_type: Optional[FunctionNode.RetType] = None) -> FunctionNode: + return self._add_child(FunctionNode, name, arguments=arguments, + return_type=return_type) + + def add_enumeration(self, name: str) -> EnumerationNode: + return self._add_child(EnumerationNode, name) + + def add_constant(self, name: str, value: str) -> ConstantNode: + return self._add_child(ConstantNode, name, value=value) + + def resolve_type_nodes(self, root: Optional[ASTNode] = None) -> None: + """Resolves type nodes for all children nodes in 2 steps: + 1. Resolve against `self` as a tree root + 2. Resolve against `root` as a tree root + Type resolution errors are postponed until all children nodes are + examined. + + Args: + root (Optional[ASTNode], optional): Root of the AST sub-tree. + Defaults to None. + """ + errors = [] + for child in itertools.chain(self.functions.values(), + self.classes.values(), + self.namespaces.values()): + try: + try: + child.resolve_type_nodes(self) # type: ignore + except TypeResolutionError: + if root is not None: + child.resolve_type_nodes(root) # type: ignore + else: + raise + except TypeResolutionError as e: + errors.append(str(e)) + if len(errors) > 0: + raise TypeResolutionError( + 'Failed to resolve "{}" namespace against "{}". ' + 'Errors: {}'.format( + self.full_export_name, + root if root is None else root.full_export_name, + errors + ) + ) diff --git a/modules/python/src2/typing_stubs_generation/nodes/node.py b/modules/python/src2/typing_stubs_generation/nodes/node.py new file mode 100644 index 0000000000..cba74c8977 --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/nodes/node.py @@ -0,0 +1,233 @@ +import abc +import enum +import itertools +from typing import (Iterator, Type, TypeVar, Iterable, Dict, + Optional, Tuple, DefaultDict) +from collections import defaultdict + +import weakref + + +ASTNodeSubtype = TypeVar("ASTNodeSubtype", bound="ASTNode") +NodeType = Type["ASTNode"] +NameToNode = Dict[str, ASTNodeSubtype] + + +class ASTNodeType(enum.Enum): + Namespace = enum.auto() + Class = enum.auto() + Function = enum.auto() + Enumeration = enum.auto() + Constant = enum.auto() + + +class ASTNode: + """Represents an element of the Abstract Syntax Tree produced by parsing + public C++ headers. + + NOTE: Every node manages a lifetime of its children nodes. Children nodes + contain only weak references to their direct parents, so there are no + circular dependencies. + """ + + def __init__(self, name: str, parent: Optional["ASTNode"] = None, + export_name: Optional[str] = None) -> None: + """ASTNode initializer + + Args: + name (str): name of the node, should be unique inside enclosing + context (There can't be 2 classes with the same name defined + in the same namespace). + parent (ASTNode, optional): parent node expressing node context. + None corresponds to globally defined object e.g. root namespace + or function without namespace. Defaults to None. + export_name (str, optional): export name of the node used to resolve + issues in languages without proper overload resolution and + provide more meaningful naming. Defaults to None. + """ + + FORBIDDEN_SYMBOLS = ";,*&#/|\\@!()[]^% " + for forbidden_symbol in FORBIDDEN_SYMBOLS: + assert forbidden_symbol not in name, \ + "Invalid node identifier '{}' - contains 1 or more "\ + "forbidden symbols: ({})".format(name, FORBIDDEN_SYMBOLS) + + assert ":" not in name, \ + "Name '{}' contains C++ scope symbols (':'). Convert the name to "\ + "Python style and create appropriate parent nodes".format(name) + + assert "." not in name, \ + "Trying to create a node with '.' symbols in its name ({}). " \ + "Dots are supposed to be a scope delimiters, so create all nodes in ('{}') " \ + "and add '{}' as a last child node".format( + name, + "->".join(name.split('.')[:-1]), + name.rsplit('.', maxsplit=1)[-1] + ) + + self.__name = name + self.export_name = name if export_name is None else export_name + self._parent: Optional["ASTNode"] = None + self.parent = parent + self.is_exported = True + self._children: DefaultDict[NodeType, NameToNode] = defaultdict(dict) + + def __str__(self) -> str: + return "{}('{}' exported as '{}')".format( + type(self).__name__.replace("Node", ""), self.name, self.export_name + ) + + def __repr__(self) -> str: + return str(self) + + @abc.abstractproperty + def children_types(self) -> Tuple[Type["ASTNode"], ...]: + """Set of ASTNode types that are allowed to be children of this node + + Returns: + Tuple[Type[ASTNode], ...]: Types of children nodes + """ + pass + + @abc.abstractproperty + def node_type(self) -> ASTNodeType: + """Type of the ASTNode that can be used to distinguish nodes without + importing all subclasses of ASTNode + + Returns: + ASTNodeType: Current node type + """ + pass + + @property + def name(self) -> str: + return self.__name + + @property + def native_name(self) -> str: + return self.full_name.replace(".", "::") + + @property + def full_name(self) -> str: + return self._construct_full_name("name") + + @property + def full_export_name(self) -> str: + return self._construct_full_name("export_name") + + @property + def parent(self) -> Optional["ASTNode"]: + return self._parent + + @parent.setter + def parent(self, value: Optional["ASTNode"]) -> None: + assert value is None or isinstance(value, ASTNode), \ + "ASTNode.parent should be None or another ASTNode, " \ + "but got: {}".format(type(value)) + + if value is not None: + value.__check_child_before_add(type(self), self.name) + + # Detach from previous parent + if self._parent is not None: + self._parent._children[type(self)].pop(self.name) + + if value is None: + self._parent = None + return + + # Set a weak reference to a new parent and add self to its children + self._parent = weakref.proxy(value) + value._children[type(self)][self.name] = self + + def __check_child_before_add(self, child_type: Type[ASTNodeSubtype], + name: str) -> None: + assert len(self.children_types) > 0, \ + "Trying to add child node '{}::{}' to node '{}::{}' " \ + "that can't have children nodes".format(child_type.__name__, name, + type(self).__name__, + self.name) + + assert child_type in self.children_types, \ + "Trying to add child node '{}::{}' to node '{}::{}' " \ + "that supports only ({}) as its children types".format( + child_type.__name__, name, type(self).__name__, self.name, + ",".join(t.__name__ for t in self.children_types) + ) + + if self._find_child(child_type, name) is not None: + raise ValueError( + "Node '{}::{}' already has a child '{}::{}'".format( + type(self).__name__, self.name, child_type.__name__, name + ) + ) + + def _add_child(self, child_type: Type[ASTNodeSubtype], name: str, + **kwargs) -> ASTNodeSubtype: + """Creates a child of the node with the given type and performs common + validation checks: + - Node can have children of the provided type + - Node doesn't have child with the same name + + NOTE: Shouldn't be used directly by a user. + + Args: + child_type (Type[ASTNodeSubtype]): Type of the child to create. + name (str): Name of the child. + **kwargs: Extra keyword arguments supplied to child_type.__init__ + method. + + Returns: + ASTNodeSubtype: Created ASTNode + """ + self.__check_child_before_add(child_type, name) + return child_type(name, parent=self, **kwargs) + + def _find_child(self, child_type: Type[ASTNodeSubtype], + name: str) -> Optional[ASTNodeSubtype]: + """Looks for child node with the given type and name. + + Args: + child_type (Type[ASTNodeSubtype]): Type of the child node. + name (str): Name of the child node. + + Returns: + Optional[ASTNodeSubtype]: child node if it can be found, None + otherwise. + """ + if child_type not in self._children: + return None + return self._children[child_type].get(name, None) + + def _construct_full_name(self, property_name: str) -> str: + """Traverses nodes hierarchy upright to the root node and constructs a + full name of the node using original or export names depending on the + provided `property_name` argument. + + Args: + property_name (str): Name of the property to quire from node to get + its name. Should be `name` or `export_name`. + + Returns: + str: full node name where each node part is divided with a dot. + """ + def get_name(node: ASTNode) -> str: + return getattr(node, property_name) + + assert property_name in ('name', 'export_name'), 'Invalid name property' + + name_parts = [get_name(self), ] + parent = self.parent + while parent is not None: + name_parts.append(get_name(parent)) + parent = parent.parent + return ".".join(reversed(name_parts)) + + def __iter__(self) -> Iterator["ASTNode"]: + return iter(itertools.chain.from_iterable( + node + # Iterate over mapping between node type and nodes dict + for children_nodes in self._children.values() + # Iterate over mapping between node name and node + for node in children_nodes.values() + )) diff --git a/modules/python/src2/typing_stubs_generation/nodes/type_node.py b/modules/python/src2/typing_stubs_generation/nodes/type_node.py new file mode 100644 index 0000000000..3a7b47ee79 --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/nodes/type_node.py @@ -0,0 +1,762 @@ +from typing import Sequence, Generator, Tuple, Optional, Union +import weakref +import abc + +from .node import ASTNode, ASTNodeType + + +class TypeResolutionError(Exception): + pass + + +class TypeNode(abc.ABC): + """This class and its derivatives used for construction parts of AST that + otherwise can't be constructed from the information provided by header + parser, because this information is either not available at that moment of + time or not available at all: + - There is no possible way to derive correspondence between C++ type + and its Python equivalent if it is not exposed from library + e.g. `cv::Rect`. + - There is no information about types visibility (see `ASTNodeTypeNode`). + """ + def __init__(self, ctype_name: str) -> None: + self.ctype_name = ctype_name + + @abc.abstractproperty + def typename(self) -> str: + """Short name of the type node used that should be used in the same + module (or a file) where type is defined. + + Returns: + str: short name of the type node. + """ + pass + + @property + def full_typename(self) -> str: + """Full name of the type node including full module name starting from + the package. + Example: 'cv2.Algorithm', 'cv2.gapi.ie.PyParams'. + + Returns: + str: full name of the type node. + """ + return self.typename + + @property + def required_definition_imports(self) -> Generator[str, None, None]: + """Generator filled with import statements required for type + node definition (especially used by `AliasTypeNode`). + + Example: + ```python + # Alias defined in the `cv2.typing.__init__.pyi` + Callback = typing.Callable[[cv2.GMat, float], None] + + # alias definition + callback_alias = AliasTypeNode.callable_( + 'Callback', + arg_types=(ASTNodeTypeNode('GMat'), PrimitiveTypeNode.float_()) + ) + + # Required definition imports + for required_import in callback_alias.required_definition_imports: + print(required_import) + # Outputs: + # 'import typing' + # 'import cv2' + ``` + + Yields: + Generator[str, None, None]: generator filled with import statements + required for type node definition. + """ + yield from () + + @property + def required_usage_imports(self) -> Generator[str, None, None]: + """Generator filled with import statements required for type node + usage. + + Example: + ```python + # Alias defined in the `cv2.typing.__init__.pyi` + Callback = typing.Callable[[cv2.GMat, float], None] + + # alias definition + callback_alias = AliasTypeNode.callable_( + 'Callback', + arg_types=(ASTNodeTypeNode('GMat'), PrimitiveTypeNode.float_()) + ) + + # Required usage imports + for required_import in callback_alias.required_usage_imports: + print(required_import) + # Outputs: + # 'import cv2.typing' + ``` + + Yields: + Generator[str, None, None]: generator filled with import statements + required for type node definition. + """ + yield from () + + @property + def is_resolved(self) -> bool: + return True + + def relative_typename(self, module_full_export_name: str) -> str: + """Type name relative to the provided module. + + Args: + module_full_export_name (str): Full export name of the module to + get relative name to. + + Returns: + str: If module name of the type node doesn't match `module`, then + returns class scopes + `self.typename`, otherwise + `self.full_typename`. + """ + return self.full_typename + + def resolve(self, root: ASTNode) -> None: + """Resolves all references to AST nodes using a top-down search + for nodes with corresponding export names. See `_resolve_symbol` for + more details. + + Args: + root (ASTNode): Node pointing to the root of a subtree in AST + representing search scope of the symbol. + Most of the symbols don't have full paths in their names, so + scopes should be examined in bottom-up manner starting + with narrowest one. + + Raises: + TypeResolutionError: if at least 1 reference to AST node can't + be resolved in the subtree pointed by the root. + """ + pass + + +class NoneTypeNode(TypeNode): + """Type node representing a None (or `void` in C++) type. + """ + @property + def typename(self) -> str: + return "None" + + +class AnyTypeNode(TypeNode): + """Type node representing any type (most of the time it means unknown). + """ + @property + def typename(self) -> str: + return "typing.Any" + + @property + def required_usage_imports(self) -> Generator[str, None, None]: + yield "import typing" + + +class PrimitiveTypeNode(TypeNode): + """Type node representing a primitive built-in types e.g. int, float, str. + """ + def __init__(self, ctype_name: str, typename: Optional[str] = None) -> None: + super().__init__(ctype_name) + self._typename = typename if typename is not None else ctype_name + + @property + def typename(self) -> str: + return self._typename + + @classmethod + def int_(cls, ctype_name: Optional[str] = None): + if ctype_name is None: + ctype_name = "int" + return PrimitiveTypeNode(ctype_name, typename="int") + + @classmethod + def float_(cls, ctype_name: Optional[str] = None): + if ctype_name is None: + ctype_name = "float" + return PrimitiveTypeNode(ctype_name, typename="float") + + @classmethod + def bool_(cls, ctype_name: Optional[str] = None): + if ctype_name is None: + ctype_name = "bool" + return PrimitiveTypeNode(ctype_name, typename="bool") + + @classmethod + def str_(cls, ctype_name: Optional[str] = None): + if ctype_name is None: + ctype_name = "string" + return PrimitiveTypeNode(ctype_name, "str") + + +class AliasRefTypeNode(TypeNode): + """Type node representing an alias referencing another alias. Example: + ```python + Point2i = tuple[int, int] + Point = Point2i + ``` + During typing stubs generation procedure above code section might be defined + as follows + ```python + AliasTypeNode.tuple_("Point2i", + items=( + PrimitiveTypeNode.int_(), + PrimitiveTypeNode.int_() + )) + AliasTypeNode.ref_("Point", "Point2i") + ``` + """ + def __init__(self, alias_ctype_name: str, + alias_export_name: Optional[str] = None): + super().__init__(alias_ctype_name) + if alias_export_name is None: + self.alias_export_name = alias_ctype_name + else: + self.alias_export_name = alias_export_name + + @property + def typename(self) -> str: + return self.alias_export_name + + @property + def full_typename(self) -> str: + return "cv2.typing." + self.typename + + +class AliasTypeNode(TypeNode): + """Type node representing an alias to another type. + Example: + ```python + Point2i = tuple[int, int] + ``` + can be defined as + ```python + AliasTypeNode.tuple_("Point2i", + items=( + PrimitiveTypeNode.int_(), + PrimitiveTypeNode.int_() + )) + ``` + Under the hood it is implemented as a container of another type node. + """ + def __init__(self, ctype_name: str, value: TypeNode, + export_name: Optional[str] = None, + comment: Optional[str] = None) -> None: + super().__init__(ctype_name) + self.value = value + self._export_name = export_name + self.comment = comment + + @property + def typename(self) -> str: + if self._export_name is not None: + return self._export_name + return self.ctype_name + + @property + def full_typename(self) -> str: + return "cv2.typing." + self.typename + + @property + def required_definition_imports(self) -> Generator[str, None, None]: + return self.value.required_usage_imports + + @property + def required_usage_imports(self) -> Generator[str, None, None]: + yield "import cv2.typing" + + @property + def is_resolved(self) -> bool: + return self.value.is_resolved + + def resolve(self, root: ASTNode): + try: + self.value.resolve(root) + except TypeResolutionError as e: + raise TypeResolutionError( + 'Failed to resolve alias "{}" exposed as "{}"'.format( + self.ctype_name, self.typename + ) + ) from e + + @classmethod + def int_(cls, ctype_name: str, export_name: Optional[str] = None, + comment: Optional[str] = None): + return cls(ctype_name, PrimitiveTypeNode.int_(), export_name, comment) + + @classmethod + def float_(cls, ctype_name: str, export_name: Optional[str] = None, + comment: Optional[str] = None): + return cls(ctype_name, PrimitiveTypeNode.float_(), export_name, comment) + + @classmethod + def array_(cls, ctype_name: str, shape: Optional[Tuple[int, ...]], + dtype: Optional[str] = None, export_name: Optional[str] = None, + comment: Optional[str] = None): + if comment is None: + comment = "Shape: " + str(shape) + else: + comment += ". Shape: " + str(shape) + return cls(ctype_name, NDArrayTypeNode(ctype_name, shape, dtype), + export_name, comment) + + @classmethod + def union_(cls, ctype_name: str, items: Tuple[TypeNode, ...], + export_name: Optional[str] = None, + comment: Optional[str] = None): + return cls(ctype_name, UnionTypeNode(ctype_name, items), + export_name, comment) + + @classmethod + def optional_(cls, ctype_name: str, item: TypeNode, + export_name: Optional[str] = None, + comment: Optional[str] = None): + return cls(ctype_name, OptionalTypeNode(item), export_name, comment) + + @classmethod + def sequence_(cls, ctype_name: str, item: TypeNode, + export_name: Optional[str] = None, + comment: Optional[str] = None): + return cls(ctype_name, SequenceTypeNode(ctype_name, item), + export_name, comment) + + @classmethod + def tuple_(cls, ctype_name: str, items: Tuple[TypeNode, ...], + export_name: Optional[str] = None, + comment: Optional[str] = None): + return cls(ctype_name, TupleTypeNode(ctype_name, items), + export_name, comment) + + @classmethod + def class_(cls, ctype_name: str, class_name: str, + export_name: Optional[str] = None, + comment: Optional[str] = None): + return cls(ctype_name, ASTNodeTypeNode(class_name), + export_name, comment) + + @classmethod + def callable_(cls, ctype_name: str, + arg_types: Union[TypeNode, Sequence[TypeNode]], + ret_type: TypeNode = NoneTypeNode("void"), + export_name: Optional[str] = None, + comment: Optional[str] = None): + return cls(ctype_name, + CallableTypeNode(ctype_name, arg_types, ret_type), + export_name, comment) + + @classmethod + def ref_(cls, ctype_name: str, alias_ctype_name: str, + alias_export_name: Optional[str] = None, + export_name: Optional[str] = None, comment: Optional[str] = None): + return cls(ctype_name, + AliasRefTypeNode(alias_ctype_name, alias_export_name), + export_name, comment) + + @classmethod + def dict_(cls, ctype_name: str, key_type: TypeNode, value_type: TypeNode, + export_name: Optional[str] = None, comment: Optional[str] = None): + return cls(ctype_name, DictTypeNode(ctype_name, key_type, value_type), + export_name, comment) + + +class NDArrayTypeNode(TypeNode): + """Type node representing NumPy ndarray. + """ + def __init__(self, ctype_name: str, shape: Optional[Tuple[int, ...]] = None, + dtype: Optional[str] = None) -> None: + super().__init__(ctype_name) + self.shape = shape + self.dtype = dtype + + @property + def typename(self) -> str: + return "numpy.ndarray[{shape}, numpy.dtype[{dtype}]]".format( + # NOTE: Shape is not fully supported yet + # shape=self.shape if self.shape is not None else "typing.Any", + shape="typing.Any", + dtype=self.dtype if self.dtype is not None else "numpy.generic" + ) + + @property + def required_usage_imports(self) -> Generator[str, None, None]: + yield "import numpy" + # if self.shape is None: + yield "import typing" + + +class ASTNodeTypeNode(TypeNode): + """Type node representing a lazy ASTNode corresponding to type of + function argument or its return type or type of class property. + Introduced laziness nature resolves the types visibility issue - all types + should be known during function declaration to select an appropriate node + from the AST. Such knowledge leads to evaluation of all preprocessor + directives (`#include` particularly) for each processed header and might be + too expensive and error prone. + """ + def __init__(self, ctype_name: str, typename: Optional[str] = None, + module_name: Optional[str] = None) -> None: + super().__init__(ctype_name) + self._typename = typename if typename is not None else ctype_name + self._module_name = module_name + self._ast_node: Optional[weakref.ProxyType[ASTNode]] = None + + @property + def typename(self) -> str: + if self._ast_node is None: + return self._typename + typename = self._ast_node.export_name + if self._ast_node.node_type is not ASTNodeType.Enumeration: + return typename + # NOTE: Special handling for enums + parent = self._ast_node.parent + while parent.node_type is ASTNodeType.Class: + typename = parent.export_name + "_" + typename + parent = parent.parent + return typename + + @property + def full_typename(self) -> str: + if self._ast_node is not None: + if self._ast_node.node_type is not ASTNodeType.Enumeration: + return self._ast_node.full_export_name + # NOTE: enumerations are exported to module scope + typename = self._ast_node.export_name + parent = self._ast_node.parent + while parent.node_type is ASTNodeType.Class: + typename = parent.export_name + "_" + typename + parent = parent.parent + return parent.full_export_name + "." + typename + if self._module_name is not None: + return self._module_name + "." + self._typename + return self._typename + + @property + def required_usage_imports(self) -> Generator[str, None, None]: + if self._module_name is None: + assert self._ast_node is not None, \ + "Can't find a module for class '{}' exported as '{}'".format( + self.ctype_name, self.typename, + ) + module = self._ast_node.parent + while module.node_type is not ASTNodeType.Namespace: + module = module.parent + yield "import " + module.full_export_name + else: + yield "import " + self._module_name + + @property + def is_resolved(self) -> bool: + return self._ast_node is not None or self._module_name is not None + + def resolve(self, root: ASTNode): + if self.is_resolved: + return + + node = _resolve_symbol(root, self.typename) + if node is None: + raise TypeResolutionError('Failed to resolve "{}" exposed as "{}"'.format( + self.ctype_name, self.typename + )) + self._ast_node = weakref.proxy(node) + + def relative_typename(self, module: str) -> str: + assert self._ast_node is not None or self._module_name is not None, \ + "'{}' exported as '{}' is not resolved yet".format(self.ctype_name, + self.typename) + if self._module_name is None: + type_module = self._ast_node.parent # type: ignore + while type_module.node_type is not ASTNodeType.Namespace: + type_module = type_module.parent + module_name = type_module.full_export_name + else: + module_name = self._module_name + if module_name != module: + return self.full_typename + return self.full_typename[len(module_name) + 1:] + + +class AggregatedTypeNode(TypeNode): + """Base type node for type nodes representing an aggregation of another + type nodes e.g. tuple, sequence or callable.""" + def __init__(self, ctype_name: str, items: Sequence[TypeNode]) -> None: + super().__init__(ctype_name) + self.items = list(items) + + @property + def is_resolved(self) -> bool: + return all(item.is_resolved for item in self.items) + + def resolve(self, root: ASTNode) -> None: + errors = [] + for item in filter(lambda item: not item.is_resolved, self): + try: + item.resolve(root) + except TypeResolutionError as e: + errors.append(str(e)) + if len(errors) > 0: + raise TypeResolutionError( + 'Failed to resolve one of "{}" items. Errors: {}'.format( + self.full_typename, errors + ) + ) + + def __iter__(self): + return iter(self.items) + + def __len__(self) -> int: + return len(self.items) + + @property + def required_definition_imports(self) -> Generator[str, None, None]: + for item in self: + yield from item.required_definition_imports + + @property + def required_usage_imports(self) -> Generator[str, None, None]: + for item in self: + yield from item.required_usage_imports + + +class ContainerTypeNode(AggregatedTypeNode): + """Base type node for all type nodes representing a container type. + """ + @property + def typename(self) -> str: + return self.type_format.format(self.types_separator.join( + item.typename for item in self + )) + + @property + def full_typename(self) -> str: + return self.type_format.format(self.types_separator.join( + item.full_typename for item in self + )) + + def relative_typename(self, module: str) -> str: + return self.type_format.format(self.types_separator.join( + item.relative_typename(module) for item in self + )) + + @abc.abstractproperty + def type_format(self) -> str: + pass + + @abc.abstractproperty + def types_separator(self) -> str: + pass + + +class SequenceTypeNode(ContainerTypeNode): + """Type node representing a homogeneous collection of elements with + possible unknown length. + """ + def __init__(self, ctype_name: str, item: TypeNode) -> None: + super().__init__(ctype_name, (item, )) + + @property + def type_format(self): + return "typing.Sequence[{}]" + + @property + def types_separator(self): + return ", " + + @property + def required_definition_imports(self) -> Generator[str, None, None]: + yield "import typing" + yield from super().required_definition_imports + + @property + def required_usage_imports(self) -> Generator[str, None, None]: + yield "import typing" + yield from super().required_usage_imports + + +class TupleTypeNode(ContainerTypeNode): + """Type node representing possibly heterogenous collection of types with + possibly unspecified length. + """ + @property + def type_format(self): + return "tuple[{}]" + + @property + def types_separator(self) -> str: + return ", " + + +class UnionTypeNode(ContainerTypeNode): + """Type node representing type that can be one of the predefined set of types. + """ + @property + def type_format(self): + return "{}" + + @property + def types_separator(self): + return " | " + + +class OptionalTypeNode(UnionTypeNode): + """Type node representing optional type which is effectively is a union + of value type node and None. + """ + def __init__(self, value: TypeNode) -> None: + super().__init__(value.ctype_name, (value, NoneTypeNode(value.ctype_name))) + + +class DictTypeNode(ContainerTypeNode): + """Type node representing a homogeneous key-value mapping. + """ + def __init__(self, ctype_name: str, key_type: TypeNode, + value_type: TypeNode) -> None: + super().__init__(ctype_name, (key_type, value_type)) + + @property + def key_type(self) -> TypeNode: + return self.items[0] + + @property + def value_type(self) -> TypeNode: + return self.items[1] + + @property + def type_format(self): + return "dict[{}]" + + @property + def types_separator(self): + return ", " + + +class CallableTypeNode(AggregatedTypeNode): + """Type node representing a callable type (most probably a function). + + ```python + CallableTypeNode( + 'image_reading_callback', + arg_types=(ASTNodeTypeNode('Image'), PrimitiveTypeNode.float_()) + ) + ``` + defines a callable type node representing a function with the same + interface as the following + ```python + def image_reading_callback(image: Image, timestamp: float) -> None: ... + ``` + """ + def __init__(self, ctype_name: str, + arg_types: Union[TypeNode, Sequence[TypeNode]], + ret_type: TypeNode = NoneTypeNode("void")) -> None: + if isinstance(arg_types, TypeNode): + super().__init__(ctype_name, (arg_types, ret_type)) + else: + super().__init__(ctype_name, (*arg_types, ret_type)) + + @property + def arg_types(self) -> Sequence[TypeNode]: + return self.items[:-1] + + @property + def ret_type(self) -> TypeNode: + return self.items[-1] + + @property + def typename(self) -> str: + return 'typing.Callable[[{}], {}]'.format( + ', '.join(arg.typename for arg in self.arg_types), + self.ret_type.typename + ) + + @property + def full_typename(self) -> str: + return 'typing.Callable[[{}], {}]'.format( + ', '.join(arg.full_typename for arg in self.arg_types), + self.ret_type.full_typename + ) + + def relative_typename(self, module: str) -> str: + return 'typing.Callable[[{}], {}]'.format( + ', '.join(arg.relative_typename(module) for arg in self.arg_types), + self.ret_type.relative_typename(module) + ) + + @property + def required_definition_imports(self) -> Generator[str, None, None]: + yield "import typing" + yield from super().required_definition_imports + + @property + def required_usage_imports(self) -> Generator[str, None, None]: + yield "import typing" + yield from super().required_usage_imports + + +def _resolve_symbol(root: Optional[ASTNode], full_symbol_name: str) -> Optional[ASTNode]: + """Searches for a symbol with the given full export name in the AST + starting from the `root`. + + Args: + root (Optional[ASTNode]): Root of the examining AST. + full_symbol_name (str): Full export name of the symbol to find. Path + components can be divided by '.' or '_'. + + Returns: + Optional[ASTNode]: ASTNode with full export name equal to + `full_symbol_name`, None otherwise. + + >>> root = NamespaceNode('cv') + >>> cls = root.add_class('Algorithm').add_class('Params') + >>> _resolve_symbol(root, 'cv.Algorithm.Params') == cls + True + + >>> root = NamespaceNode('cv') + >>> enum = root.add_namespace('detail').add_enumeration('AlgorithmType') + >>> _resolve_symbol(root, 'cv_detail_AlgorithmType') == enum + True + + >>> root = NamespaceNode('cv') + >>> _resolve_symbol(root, 'cv.detail.Algorithm') + None + + >>> root = NamespaceNode('cv') + >>> enum = root.add_namespace('detail').add_enumeration('AlgorithmType') + >>> _resolve_symbol(root, 'AlgorithmType') + None + """ + def search_down_symbol(scope: Optional[ASTNode], + scope_sep: str) -> Optional[ASTNode]: + parts = full_symbol_name.split(scope_sep, maxsplit=1) + while len(parts) == 2: + # Try to find narrow scope + scope = _resolve_symbol(scope, parts[0]) + if scope is None: + return None + # and resolve symbol in it + node = _resolve_symbol(scope, parts[1]) + if node is not None: + return node + # symbol is not found, but narrowed scope is valid - diving further + parts = parts[1].split(scope_sep, maxsplit=1) + return None + + assert root is not None, \ + "Can't resolve symbol '{}' from NONE root".format(full_symbol_name) + # Looking for exact symbol match + for attr in filter(lambda attr: hasattr(root, attr), + ("namespaces", "classes", "enumerations")): + nodes_dict = getattr(root, attr) + node = nodes_dict.get(full_symbol_name, None) + if node is not None: + return node + # Symbol is not found, looking for more fine-grained scope if possible + for scope_sep in ("_", "."): + node = search_down_symbol(root, scope_sep) + if node is not None: + return node + return None diff --git a/modules/python/src2/typing_stubs_generation/predefined_types.py b/modules/python/src2/typing_stubs_generation/predefined_types.py new file mode 100644 index 0000000000..36f0eac9eb --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/predefined_types.py @@ -0,0 +1,195 @@ +from .nodes.type_node import ( + AliasTypeNode, AliasRefTypeNode, PrimitiveTypeNode, + ASTNodeTypeNode, NDArrayTypeNode, NoneTypeNode, SequenceTypeNode, + TupleTypeNode, UnionTypeNode, AnyTypeNode +) + +# Set of predefined types used to cover cases when library doesn't +# directly exports a type and equivalent one should be used instead. +# Example: Instead of C++ `cv::Rect(1, 1, 5, 6)` in Python any sequence type +# with length 4 can be used: tuple `(1, 1, 5, 6)` or list `[1, 1, 5, 6]`. +# Predefined type might be: +# - alias - defines a Python synonym for a native type name. +# Example: `cv::Rect` and `cv::Size` are both `Sequence[int]` in Python, but +# with different length constraints (4 and 2 accordingly). +# - direct substitution - just a plain type replacement without any credits to +# native type. Example: +# * `std::vector` is `np.ndarray` with `dtype == np.uint8` in Python +# * `double` is a Python `float` +# * `std::string` is a Python `str` +_PREDEFINED_TYPES = ( + PrimitiveTypeNode.int_("int"), + PrimitiveTypeNode.int_("uchar"), + PrimitiveTypeNode.int_("unsigned"), + PrimitiveTypeNode.int_("int64"), + PrimitiveTypeNode.int_("size_t"), + PrimitiveTypeNode.float_("float"), + PrimitiveTypeNode.float_("double"), + PrimitiveTypeNode.bool_("bool"), + PrimitiveTypeNode.str_("string"), + PrimitiveTypeNode.str_("char"), + PrimitiveTypeNode.str_("String"), + PrimitiveTypeNode.str_("c_string"), + NoneTypeNode("void"), + AliasTypeNode.int_("void*", "IntPointer", "Represents an arbitrary pointer"), + AliasTypeNode.union_( + "Mat", + items=(ASTNodeTypeNode("Mat", module_name="cv2.mat_wrapper"), + NDArrayTypeNode("Mat")), + export_name="MatLike" + ), + AliasTypeNode.sequence_("MatShape", PrimitiveTypeNode.int_()), + AliasTypeNode.sequence_("Size", PrimitiveTypeNode.int_(), + comment="Required length is 2"), + AliasTypeNode.sequence_("Size2f", PrimitiveTypeNode.float_(), + comment="Required length is 2"), + AliasTypeNode.sequence_("Scalar", PrimitiveTypeNode.float_(), + comment="Required length is at most 4"), + AliasTypeNode.sequence_("Point", PrimitiveTypeNode.int_(), + comment="Required length is 2"), + AliasTypeNode.ref_("Point2i", "Point"), + AliasTypeNode.sequence_("Point2f", PrimitiveTypeNode.float_(), + comment="Required length is 2"), + AliasTypeNode.sequence_("Point2d", PrimitiveTypeNode.float_(), + comment="Required length is 2"), + AliasTypeNode.sequence_("Point3i", PrimitiveTypeNode.int_(), + comment="Required length is 3"), + AliasTypeNode.sequence_("Point3f", PrimitiveTypeNode.float_(), + comment="Required length is 3"), + AliasTypeNode.sequence_("Point3d", PrimitiveTypeNode.float_(), + comment="Required length is 3"), + AliasTypeNode.sequence_("Range", PrimitiveTypeNode.int_(), + comment="Required length is 2"), + AliasTypeNode.sequence_("Rect", PrimitiveTypeNode.int_(), + comment="Required length is 4"), + AliasTypeNode.sequence_("Rect2i", PrimitiveTypeNode.int_(), + comment="Required length is 4"), + AliasTypeNode.sequence_("Rect2d", PrimitiveTypeNode.float_(), + comment="Required length is 4"), + AliasTypeNode.dict_("Moments", PrimitiveTypeNode.str_("Moments::key"), + PrimitiveTypeNode.float_("Moments::value")), + AliasTypeNode.tuple_("RotatedRect", + items=(AliasRefTypeNode("Point2f"), + AliasRefTypeNode("Size"), + PrimitiveTypeNode.float_()), + comment="Any type providing sequence protocol is supported"), + AliasTypeNode.tuple_("TermCriteria", + items=( + ASTNodeTypeNode("TermCriteria.Type"), + PrimitiveTypeNode.int_(), + PrimitiveTypeNode.float_()), + comment="Any type providing sequence protocol is supported"), + AliasTypeNode.sequence_("Vec2i", PrimitiveTypeNode.int_(), + comment="Required length is 2"), + AliasTypeNode.sequence_("Vec2f", PrimitiveTypeNode.float_(), + comment="Required length is 2"), + AliasTypeNode.sequence_("Vec2d", PrimitiveTypeNode.float_(), + comment="Required length is 2"), + AliasTypeNode.sequence_("Vec3i", PrimitiveTypeNode.int_(), + comment="Required length is 3"), + AliasTypeNode.sequence_("Vec3f", PrimitiveTypeNode.float_(), + comment="Required length is 3"), + AliasTypeNode.sequence_("Vec3d", PrimitiveTypeNode.float_(), + comment="Required length is 3"), + AliasTypeNode.sequence_("Vec4i", PrimitiveTypeNode.int_(), + comment="Required length is 4"), + AliasTypeNode.sequence_("Vec4f", PrimitiveTypeNode.float_(), + comment="Required length is 4"), + AliasTypeNode.sequence_("Vec4d", PrimitiveTypeNode.float_(), + comment="Required length is 4"), + AliasTypeNode.sequence_("Vec6f", PrimitiveTypeNode.float_(), + comment="Required length is 6"), + AliasTypeNode.class_("FeatureDetector", "Feature2D", + export_name="FeatureDetector"), + AliasTypeNode.class_("DescriptorExtractor", "Feature2D", + export_name="DescriptorExtractor"), + AliasTypeNode.class_("FeatureExtractor", "Feature2D", + export_name="FeatureExtractor"), + AliasTypeNode.union_("GProtoArg", + items=(AliasRefTypeNode("Scalar"), + ASTNodeTypeNode("GMat"), + ASTNodeTypeNode("GOpaqueT"), + ASTNodeTypeNode("GArrayT"))), + SequenceTypeNode("GProtoArgs", AliasRefTypeNode("GProtoArg")), + AliasTypeNode.sequence_("GProtoInputArgs", AliasRefTypeNode("GProtoArg")), + AliasTypeNode.sequence_("GProtoOutputArgs", AliasRefTypeNode("GProtoArg")), + AliasTypeNode.union_( + "GRunArg", + items=(AliasRefTypeNode("Mat", "MatLike"), + AliasRefTypeNode("Scalar"), + ASTNodeTypeNode("GOpaqueT"), + ASTNodeTypeNode("GArrayT"), + SequenceTypeNode("GRunArg", AnyTypeNode("GRunArg")), + NoneTypeNode("GRunArg")) + ), + AliasTypeNode.optional_("GOptRunArg", AliasRefTypeNode("GRunArg")), + AliasTypeNode.union_("GMetaArg", + items=(ASTNodeTypeNode("GMat"), + AliasRefTypeNode("Scalar"), + ASTNodeTypeNode("GOpaqueT"), + ASTNodeTypeNode("GArrayT"))), + AliasTypeNode.union_("Prim", + items=(ASTNodeTypeNode("gapi.wip.draw.Text"), + ASTNodeTypeNode("gapi.wip.draw.Circle"), + ASTNodeTypeNode("gapi.wip.draw.Image"), + ASTNodeTypeNode("gapi.wip.draw.Line"), + ASTNodeTypeNode("gapi.wip.draw.Rect"), + ASTNodeTypeNode("gapi.wip.draw.Mosaic"), + ASTNodeTypeNode("gapi.wip.draw.Poly"))), + SequenceTypeNode("Prims", AliasRefTypeNode("Prim")), + AliasTypeNode.array_("Matx33f", (3, 3), "numpy.float32"), + AliasTypeNode.array_("Matx33d", (3, 3), "numpy.float64"), + AliasTypeNode.array_("Matx44f", (4, 4), "numpy.float32"), + AliasTypeNode.array_("Matx44d", (4, 4), "numpy.float64"), + NDArrayTypeNode("vector", dtype="numpy.uint8"), + NDArrayTypeNode("vector_uchar", dtype="numpy.uint8"), + TupleTypeNode("GMat2", items=(ASTNodeTypeNode("GMat"), + ASTNodeTypeNode("GMat"))), + ASTNodeTypeNode("GOpaque", "GOpaqueT"), + ASTNodeTypeNode("GArray", "GArrayT"), + AliasTypeNode.union_("GTypeInfo", + items=(ASTNodeTypeNode("GMat"), + AliasRefTypeNode("Scalar"), + ASTNodeTypeNode("GOpaqueT"), + ASTNodeTypeNode("GArrayT"))), + SequenceTypeNode("GCompileArgs", ASTNodeTypeNode("GCompileArg")), + SequenceTypeNode("GTypesInfo", AliasRefTypeNode("GTypeInfo")), + SequenceTypeNode("GRunArgs", AliasRefTypeNode("GRunArg")), + SequenceTypeNode("GMetaArgs", AliasRefTypeNode("GMetaArg")), + SequenceTypeNode("GOptRunArgs", AliasRefTypeNode("GOptRunArg")), + AliasTypeNode.callable_( + "detail_ExtractArgsCallback", + arg_types=SequenceTypeNode("GTypesInfo", AliasRefTypeNode("GTypeInfo")), + ret_type=SequenceTypeNode("GRunArgs", AliasRefTypeNode("GRunArg")), + export_name="ExtractArgsCallback" + ), + AliasTypeNode.callable_( + "detail_ExtractMetaCallback", + arg_types=SequenceTypeNode("GTypesInfo", AliasRefTypeNode("GTypeInfo")), + ret_type=SequenceTypeNode("GMetaArgs", AliasRefTypeNode("GMetaArg")), + export_name="ExtractMetaCallback" + ), + AliasTypeNode.class_("LayerId", "DictValue"), + PrimitiveTypeNode.int_("cvflann_flann_distance_t"), + PrimitiveTypeNode.int_("flann_flann_distance_t"), + PrimitiveTypeNode.int_("cvflann_flann_algorithm_t"), + PrimitiveTypeNode.int_("flann_flann_algorithm_t"), + AliasTypeNode.dict_("flann_IndexParams", + key_type=PrimitiveTypeNode.str_(), + value_type=UnionTypeNode("flann_IndexParams::value", items=( + PrimitiveTypeNode.bool_(), + PrimitiveTypeNode.int_(), + PrimitiveTypeNode.float_(), + PrimitiveTypeNode.str_()) + ), export_name="IndexParams"), + AliasTypeNode.dict_("flann_SearchParams", + key_type=PrimitiveTypeNode.str_(), + value_type=UnionTypeNode("flann_IndexParams::value", items=( + PrimitiveTypeNode.bool_(), + PrimitiveTypeNode.int_(), + PrimitiveTypeNode.float_(), + PrimitiveTypeNode.str_()) + ), export_name="SearchParams"), +) + +PREDEFINED_TYPES = dict(zip((t.ctype_name for t in _PREDEFINED_TYPES), _PREDEFINED_TYPES)) diff --git a/modules/python/src2/typing_stubs_generation/types_conversion.py b/modules/python/src2/typing_stubs_generation/types_conversion.py new file mode 100644 index 0000000000..fdbd7208a4 --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/types_conversion.py @@ -0,0 +1,333 @@ +from typing import Tuple, List, Optional + +from .predefined_types import PREDEFINED_TYPES +from .nodes.type_node import ( + TypeNode, UnionTypeNode, SequenceTypeNode, ASTNodeTypeNode, TupleTypeNode +) + + +def replace_template_parameters_with_placeholders(string: str) \ + -> Tuple[str, Tuple[str, ...]]: + """Replaces template parameters with `format` placeholders for all template + instantiations in provided string. + Only outermost template parameters are replaced. + + Args: + string (str): input string containing C++ template instantiations + + Returns: + tuple[str, tuple[str, ...]]: string with '{}' placeholders template + instead of instantiation types and a tuple of extracted types. + + >>> template_string, args = replace_template_parameters_with_placeholders( + ... "std::vector>, test" + ... ) + >>> template_string.format(*args) == "std::vector>, test" + True + + >>> replace_template_parameters_with_placeholders( + ... "cv::util::variant" + ... ) + ('cv::util::variant<{}>', ('cv::GRunArgs, cv::GOptRunArgs',)) + + >>> replace_template_parameters_with_placeholders("vector>") + ('vector<{}>', ('Point',)) + + >>> replace_template_parameters_with_placeholders( + ... "vector>, vector" + ... ) + ('vector<{}>, vector<{}>', ('Point', 'float')) + + >>> replace_template_parameters_with_placeholders("string without templates") + ('string without templates', ()) + """ + + template_brackets_indices = [] + template_instantiations_count = 0 + template_start_index = 0 + for i, c in enumerate(string): + if c == "<": + template_instantiations_count += 1 + if template_instantiations_count == 1: + # + 1 - because left bound is included in substring range + template_start_index = i + 1 + elif c == ">": + template_instantiations_count -= 1 + assert template_instantiations_count >= 0, \ + "Provided string is ill-formed. There are more '>' than '<'." + if template_instantiations_count == 0: + template_brackets_indices.append((template_start_index, i)) + assert template_instantiations_count == 0, \ + "Provided string is ill-formed. There are more '<' than '>'." + template_args: List[str] = [] + # Reversed loop is required to preserve template start/end indices + for i, j in reversed(template_brackets_indices): + template_args.insert(0, string[i:j]) + string = string[:i] + "{}" + string[j:] + return string, tuple(template_args) + + +def get_template_instantiation_type(typename: str) -> str: + """Extracts outermost template instantiation type from provided string + + Args: + typename (str): String containing C++ template instantiation. + + Returns: + str: String containing template instantiation type + + >>> get_template_instantiation_type("std::vector>") + 'cv::Point' + >>> get_template_instantiation_type("std::vector") + 'uchar' + >>> get_template_instantiation_type("std::map") + 'int, float' + >>> get_template_instantiation_type("uchar") + Traceback (most recent call last): + ... + ValueError: typename ('uchar') doesn't contain template instantiations + >>> get_template_instantiation_type("std::vector, std::vector") + Traceback (most recent call last): + ... + ValueError: typename ('std::vector, std::vector') contains more than 1 template instantiation + """ + + _, args = replace_template_parameters_with_placeholders(typename) + if len(args) == 0: + raise ValueError( + "typename ('{}') doesn't contain template instantiations".format(typename) + ) + if len(args) > 1: + raise ValueError( + "typename ('{}') contains more than 1 template instantiation".format(typename) + ) + return args[0] + + +def normalize_ctype_name(typename: str) -> str: + """Normalizes C++ name by removing unnecessary namespace prefixes and possible + pointer/reference qualification. '::' are replaced with '_'. + + NOTE: Pointer decay for 'void*' is not performed. + + Args: + typename (str): Name of the C++ type for normalization + + Returns: + str: Normalized C++ type name. + + >>> normalize_ctype_name('std::vector&') + 'vector' + >>> normalize_ctype_name('AKAZE::DescriptorType') + 'AKAZE_DescriptorType' + >>> normalize_ctype_name('std::vector') + 'vector' + >>> normalize_ctype_name('std::string') + 'string' + >>> normalize_ctype_name('void*') # keep void* as is - special case + 'void*' + >>> normalize_ctype_name('Ptr') + 'AKAZE' + >>> normalize_ctype_name('Algorithm_Ptr') + 'Algorithm' + """ + for prefix_to_remove in ("cv", "std"): + if typename.startswith(prefix_to_remove): + typename = typename[len(prefix_to_remove):] + typename = typename.replace("::", "_").lstrip("_") + if typename.endswith('&'): + typename = typename[:-1] + typename = typename.strip() + + if typename == 'void*': + return typename + + if is_pointer_type(typename): + # Case for "type*", "type_Ptr", "typePtr" + for suffix in ("*", "_Ptr", "Ptr"): + if typename.endswith(suffix): + return typename[:-len(suffix)] + # Case Ptr + if _is_template_instantiation(typename): + return normalize_ctype_name( + get_template_instantiation_type(typename) + ) + # Case Ptr_Type + return typename.split("_", maxsplit=1)[-1] + + # special normalization for several G-API Types + if typename.startswith("GArray_") or typename.startswith("GArray<"): + return "GArrayT" + if typename.startswith("GOpaque_") or typename.startswith("GOpaque<"): + return "GOpaqueT" + if typename == "GStreamerPipeline" or typename.startswith("GStreamerSource"): + return "gst_" + typename + + return typename + + +def is_tuple_type(typename: str) -> bool: + return typename.startswith("tuple") or typename.startswith("pair") + + +def is_sequence_type(typename: str) -> bool: + return typename.startswith("vector") + + +def is_pointer_type(typename: str) -> bool: + return typename.endswith("Ptr") or typename.endswith("*") \ + or typename.startswith("Ptr") + + +def is_union_type(typename: str) -> bool: + return typename.startswith('util_variant') + + +def _is_template_instantiation(typename: str) -> bool: + """Fast, but unreliable check whenever provided typename is a template + instantiation. + + Args: + typename (str): typename to check against template instantiation. + + Returns: + bool: True if provided `typename` contains template instantiation, + False otherwise + """ + + if "<" in typename: + assert ">" in typename, \ + "Wrong template class instantiation: {}. '>' is missing".format(typename) + return True + return False + + +def create_type_nodes_from_template_arguments(template_args_str: str) \ + -> List[TypeNode]: + """Creates a list of type nodes corresponding to the argument types + used for template instantiation. + This method correctly addresses the situation when arguments of the input + template are also templates. + Example: + if `create_type_node` is called with + `std::tuple, int, std::vector>` + this function will be called with + `std::variant>, int, std::vector` + that produces the following order of types resolution + `std::variant` ~ `Union` + `std::variant` -> `int` ~ `int` -> `Union[int, Point2i]` + `Point2i` ~ `Point2i` + `int` -> `int` + `std::vector` -> `std::vector` ~ `Sequence` -> `Sequence[int]` + `int` ~ `int` + + Returns: + List[TypeNode]: set of type nodes used for template instantiation. + List is empty if input string doesn't contain template instantiation. + """ + + type_nodes = [] + template_args_str, templated_args_types = replace_template_parameters_with_placeholders( + template_args_str + ) + template_index = 0 + # For each template argument + for template_arg in template_args_str.split(","): + template_arg = template_arg.strip() + # Check if argument requires type substitution + if _is_template_instantiation(template_arg): + # Reconstruct the original type + template_arg = template_arg.format(templated_args_types[template_index]) + template_index += 1 + # create corresponding type node + type_nodes.append(create_type_node(template_arg)) + return type_nodes + + +def create_type_node(typename: str, + original_ctype_name: Optional[str] = None) -> TypeNode: + """Converts C++ type name to appropriate type used in Python library API. + + Conversion procedure: + 1. Normalize typename: remove redundant prefixes, unify name + components delimiters, remove reference qualifications. + 2. Check whenever typename has a known predefined conversion or exported + as alias e.g. + - C++ `double` -> Python `float` + - C++ `cv::Rect` -> Python `Sequence[int]` + - C++ `std::vector` -> Python `np.ndarray` + return TypeNode corresponding to the appropriate type. + 3. Check whenever typename is a container of types e.g. variant, + sequence or tuple. If so, select appropriate Python container type + and perform arguments conversion. + 4. Create a type node corresponding to the AST node passing normalized + typename as its name. + + Args: + typename (str): C++ type name to convert. + original_ctype_name (Optional[str]): Original C++ name of the type. + `original_ctype_name` == `typename` if provided argument is None. + Default is None. + + Returns: + TypeNode: type node that wraps C++ type exposed to Python + + >>> create_type_node('Ptr').typename + 'AKAZE' + >>> create_type_node('std::vector>').typename + 'typing.Sequence[Algorithm]' + """ + + if original_ctype_name is None: + original_ctype_name = typename + + typename = normalize_ctype_name(typename.strip()) + + # if typename is a known alias or has explicitly defined substitution + type_node = PREDEFINED_TYPES.get(typename) + if type_node is not None: + type_node.ctype_name = original_ctype_name + return type_node + + # If typename is a known exported alias name (e.g. IndexParams or SearchParams) + for alias in PREDEFINED_TYPES.values(): + if alias.typename == typename: + return alias + + if is_union_type(typename): + union_types = get_template_instantiation_type(typename) + return UnionTypeNode( + original_ctype_name, + items=create_type_nodes_from_template_arguments(union_types) + ) + + # if typename refers to a sequence type e.g. vector + if is_sequence_type(typename): + # Recursively convert sequence element type + if _is_template_instantiation(typename): + inner_sequence_type = create_type_node( + get_template_instantiation_type(typename) + ) + else: + # Handle vector_Type cases + # maxsplit=1 is required to handle sequence of sequence e.g: + # vector_vector_Mat -> Sequence[Sequence[Mat]] + inner_sequence_type = create_type_node(typename.split("_", 1)[-1]) + return SequenceTypeNode(original_ctype_name, inner_sequence_type) + + # If typename refers to a heterogeneous container + # (can contain elements of different types) + if is_tuple_type(typename): + tuple_types = get_template_instantiation_type(typename) + return TupleTypeNode( + original_ctype_name, + items=create_type_nodes_from_template_arguments(tuple_types) + ) + # If everything else is False, it means that input typename refers to a + # class or enum of the library. + return ASTNodeTypeNode(original_ctype_name, typename) + + +if __name__ == "__main__": + import doctest + doctest.testmod() diff --git a/modules/python/src2/typing_stubs_generator.py b/modules/python/src2/typing_stubs_generator.py new file mode 100644 index 0000000000..fa0b2b7a28 --- /dev/null +++ b/modules/python/src2/typing_stubs_generator.py @@ -0,0 +1,173 @@ +"""Contains a class used to resolve compatibility issues with old Python versions. + +Typing stubs generation is available starting from Python 3.6 only. +For other versions all calls to functions are noop. +""" + +import sys +import warnings + + +if sys.version_info >= (3, 6): + from contextlib import contextmanager + + from typing import Dict, Set, Any, Sequence, Generator, Union + + from pathlib import Path + + from typing_stubs_generation import ( + generate_typing_stubs, + NamespaceNode, + EnumerationNode, + SymbolName, + ClassNode, + create_function_node, + create_class_node, + find_class_node, + resolve_enum_scopes + ) + + import functools + + class FailuresWrapper: + def __init__(self, exceptions_as_warnings=True): + self.has_failure = False + self.exceptions_as_warnings = exceptions_as_warnings + + def wrap_exceptions_as_warnings(self, original_func=None, + ret_type_on_failure=None): + def parametrized_wrapper(func): + @functools.wraps(func) + def wrapped_func(*args, **kwargs): + if self.has_failure: + if ret_type_on_failure is None: + return None + return ret_type_on_failure() + + try: + ret_type = func(*args, **kwargs) + except Exception as e: + self.has_failure = True + warnings.warn( + 'Typing stubs generation has failed. Reason: {}'.format(e) + ) + if ret_type_on_failure is None: + return None + return ret_type_on_failure() + return ret_type + + if self.exceptions_as_warnings: + return wrapped_func + else: + return original_func + + if original_func: + return parametrized_wrapper(original_func) + return parametrized_wrapper + + @contextmanager + def delete_on_failure(self, file_path): + # type: (Path) -> Generator[None, None, None] + # There is no errors during stubs generation and file doesn't exist + if not self.has_failure and not file_path.is_file(): + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.touch() + try: + # continue execution + yield + finally: + # If failure is occurred - delete file if exists + if self.has_failure and file_path.is_file(): + file_path.unlink() + + failures_wrapper = FailuresWrapper(exceptions_as_warnings=True) + + class ClassNodeStub: + def add_base(self, base_node): + pass + + class TypingStubsGenerator: + def __init__(self): + self.cv_root = NamespaceNode("cv", export_name="cv2") + self.exported_enums = {} # type: Dict[SymbolName, EnumerationNode] + self.type_hints_ignored_functions = set() # type: Set[str] + + @failures_wrapper.wrap_exceptions_as_warnings + def add_enum(self, symbol_name, is_scoped_enum, entries): + # type: (SymbolName, bool, Dict[str, str]) -> None + if symbol_name in self.exported_enums: + assert symbol_name.name == "", \ + "Trying to export 2 enums with same symbol " \ + "name: {}".format(symbol_name) + enumeration_node = self.exported_enums[symbol_name] + else: + enumeration_node = EnumerationNode(symbol_name.name, + is_scoped_enum) + self.exported_enums[symbol_name] = enumeration_node + for entry_name, entry_value in entries.items(): + enumeration_node.add_constant(entry_name, entry_value) + + @failures_wrapper.wrap_exceptions_as_warnings + def add_ignored_function_name(self, function_name): + # type: (str) -> None + self.type_hints_ignored_functions.add(function_name) + + @failures_wrapper.wrap_exceptions_as_warnings + def create_function_node(self, func_info): + # type: (Any) -> None + create_function_node(self.cv_root, func_info) + + @failures_wrapper.wrap_exceptions_as_warnings(ret_type_on_failure=ClassNodeStub) + def find_class_node(self, class_info, namespaces): + # type: (Any, Sequence[str]) -> ClassNode + return find_class_node(self.cv_root, class_info.full_original_name, namespaces) + + @failures_wrapper.wrap_exceptions_as_warnings(ret_type_on_failure=ClassNodeStub) + def create_class_node(self, class_info, namespaces): + # type: (Any, Sequence[str]) -> ClassNode + return create_class_node(self.cv_root, class_info, namespaces) + + def generate(self, output_path): + # type: (Union[str, Path]) -> None + output_path = Path(output_path) + py_typed_path = output_path / self.cv_root.export_name / 'py.typed' + with failures_wrapper.delete_on_failure(py_typed_path): + self._generate(output_path) + + @failures_wrapper.wrap_exceptions_as_warnings + def _generate(self, output_path): + # type: (Path) -> None + resolve_enum_scopes(self.cv_root, self.exported_enums) + generate_typing_stubs(self.cv_root, output_path) + + +else: + class ClassNode: + def add_base(self, base_node): + pass + + class TypingStubsGenerator: + def __init__(self): + self.type_hints_ignored_functions = set() # type: Set[str] + print( + 'WARNING! Typing stubs can be generated only with Python 3.6 or higher. ' + 'Current version {}'.format(sys.version_info) + ) + + def add_enum(self, symbol_name, is_scoped_enum, entries): + pass + + def add_ignored_function_name(self, function_name): + pass + + def create_function_node(self, func_info): + pass + + def create_class_node(self, class_info, namespaces): + return ClassNode() + + def find_class_node(self, class_info, namespaces): + return ClassNode() + + def generate(self, output_path): + pass