potok/opencv
1
0
Fork 0
mirror of https://github.com/opencv/opencv.git synced 2026-07-29 23:33:05 +04:00
Code Packages Releases Wiki Activity

Merge pull request #20370 from ddacw:stub-gen-next

Browse Source 
Python typing stub generation #20370

Add stub generation to `gen2.py`, addressing #14590.

### Pull Request Readiness Checklist

See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request

- [x] I agree to contribute to the project under Apache 2 License.
- [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or other license that is incompatible with OpenCV
- [x] The PR is proposed to proper branch
- [x] There is reference to original bug report and related work
- [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable
      Patch to opencv_extra has the same branch name.
- [ ] The feature is well documented and sample code can be built with the project CMake
This commit is contained in:
Duong Dac
2023-05-26 17:25:46 +02:00
committed by GitHub
parent cf0ba039c3
commit a9424868a1
21 changed files with 3449 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/python/src2/gen2.py
+94 -35
View File
@@ -1,16 +1,34 @@
#!/usr/bin/env python
from __future__ import print_function
import hdr_parser, sys, re, os
import hdr_parser, sys, re
from string import Template
from pprint import pprint
from collections import namedtuple
from itertools import chain
from typing_stubs_generator import TypingStubsGenerator
if sys.version_info[0] >= 3:
from io import StringIO
else:
from cStringIO import StringIO
if sys.version_info >= (3, 6):
from typing_stubs_generation import SymbolName
else:
SymbolName = namedtuple('SymbolName', ('namespaces', 'classes', 'name'))
def parse_symbol_name(cls, full_symbol_name, known_namespaces):
chunks = full_symbol_name.split('.')
namespaces, name = chunks[:-1], chunks[-1]
classes = []
while len(namespaces) > 0 and '.'.join(namespaces) not in known_namespaces:
classes.insert(0, namespaces.pop())
return cls(tuple(namespaces), tuple(classes), name)
setattr(SymbolName, "parse", classmethod(parse_symbol_name))
forbidden_arg_types = ["void*"]
@@ -182,6 +200,7 @@ ${variant}
}
""")
class FormatStrings:
string = 's'
unsigned_char = 'b'
@@ -197,9 +216,9 @@ class FormatStrings:
double = 'd'
object = 'O'
ArgTypeInfo = namedtuple('ArgTypeInfo',
['atype', 'format_str', 'default_value',
'strict_conversion'])
['atype', 'format_str', 'default_value', 'strict_conversion'])
# strict_conversion is False by default
ArgTypeInfo.__new__.__defaults__ = (False,)
@@ -256,12 +275,16 @@ class ClassProp(object):
class ClassInfo(object):
def __init__(self, name, decl=None, codegen=None):
# Scope name can be a module or other class e.g. cv::SimpleBlobDetector::Params
scope_name, self.original_name = name.rsplit(".", 1)
self.original_scope_name, self.original_name = name.rsplit(".", 1)
# In case scope refer the outer class exported with different name
if codegen:
scope_name = codegen.get_export_scope_name(scope_name)
self.scope_name = re.sub(r"^cv\.?", "", scope_name)
self.export_scope_name = codegen.get_export_scope_name(
self.original_scope_name
)
else:
self.export_scope_name = self.original_scope_name
self.export_scope_name = re.sub(r"^cv\.?", "", self.export_scope_name)
self.export_name = self.original_name
@@ -312,8 +335,8 @@ class ClassInfo(object):
@property
def wname(self):
if len(self.scope_name) > 0:
return self.scope_name.replace(".", "_") + "_" + self.export_name
if len(self.export_scope_name) > 0:
return self.export_scope_name.replace(".", "_") + "_" + self.export_name
return self.export_name
@@ -322,16 +345,16 @@ class ClassInfo(object):
return self.class_id
@property
def full_scope_name(self):
return "cv." + self.scope_name if len(self.scope_name) else "cv"
def full_export_scope_name(self):
return "cv." + self.export_scope_name if len(self.export_scope_name) else "cv"
@property
def full_export_name(self):
return self.full_scope_name + "." + self.export_name
return self.full_export_scope_name + "." + self.export_name
@property
def full_original_name(self):
return self.full_scope_name + "." + self.original_name
return self.original_scope_name + "." + self.original_name
@property
def has_export_alias(self):
@@ -415,7 +438,7 @@ class ClassInfo(object):
baseptr,
constructor_name,
# Leading dot is required to provide correct class naming
"." + self.scope_name if len(self.scope_name) > 0 else self.scope_name
"." + self.export_scope_name if len(self.export_scope_name) > 0 else self.export_scope_name
)
@@ -577,6 +600,10 @@ class FuncVariant(object):
self.args.append(ainfo)
self.init_pyproto(namespace, classname, known_classes)
def is_arg_optional(self, py_arg_index):
# type: (FuncVariant, int) -> bool
return py_arg_index >= len(self.py_arglist) - self.py_noptargs
def init_pyproto(self, namespace, classname, known_classes):
# string representation of argument list, with '[', ']' symbols denoting optional arguments, e.g.
# "src1, src2[, dst[, mask]]" for cv.add
@@ -1053,6 +1080,7 @@ class PythonWrapperGenerator(object):
self.namespaces = {}
self.consts = {}
self.enums = {}
self.typing_stubs_generator = TypingStubsGenerator()
self.code_include = StringIO()
self.code_enums = StringIO()
self.code_types = StringIO()
@@ -1099,26 +1127,19 @@ class PythonWrapperGenerator(object):
return original_scope_name
def split_decl_name(self, name):
chunks = name.split('.')
namespace = chunks[:-1]
classes = []
while namespace and '.'.join(namespace) not in self.parser.namespaces:
classes.insert(0, namespace.pop())
return namespace, classes, chunks[-1]
return SymbolName.parse(name, self.parser.namespaces)
def add_const(self, name, decl):
cname = name.replace('.','::')
namespace, classes, name = self.split_decl_name(name)
namespace = '.'.join(namespace)
name = '_'.join(classes+[name])
name = '_'.join(chain(classes, (name, )))
ns = self.namespaces.setdefault(namespace, Namespace())
if name in ns.consts:
print("Generator error: constant %s (cname=%s) already exists" \
% (name, cname))
sys.exit(-1)
ns.consts[name] = cname
value = decl[1]
py_name = '.'.join([namespace, name])
py_signatures = self.py_signatures.setdefault(cname, [])
@@ -1126,20 +1147,31 @@ class PythonWrapperGenerator(object):
#print(cname + ' => ' + str(py_name) + ' (value=' + value + ')')
def add_enum(self, name, decl):
enumeration_name = SymbolName.parse(name, self.parser.namespaces)
is_scoped_enum = decl[0].startswith("enum class") \
or decl[0].startswith("enum struct")
wname = normalize_class_name(name)
if wname.endswith("<unnamed>"):
wname = None
else:
self.enums[wname] = name
const_decls = decl[3]
enum_entries = {}
for decl in const_decls:
name = decl[0]
self.add_const(name.replace("const ", "").strip(), decl)
enum_entries[decl[0].split(".")[-1]] = decl[1]
self.add_const(decl[0].replace("const ", "").strip(), decl)
# Extra enumerations tracking is required to generate stubs for
# all enumerations, including <unnamed> once, otherwise they
# will be forgiven
self.typing_stubs_generator.add_enum(enumeration_name, is_scoped_enum,
enum_entries)
def add_func(self, decl):
namespace, classes, barename = self.split_decl_name(decl[0])
cname = "::".join(namespace+classes+[barename])
cname = "::".join(chain(namespace, classes, (barename, )))
name = barename
classname = ''
bareclassname = ''
@@ -1166,7 +1198,7 @@ class PythonWrapperGenerator(object):
return
if isconstructor:
name = "_".join(classes[:-1]+[name])
name = "_".join(chain(classes[:-1], (name, )))
if is_static:
# Add it as a method to the class
@@ -1175,7 +1207,7 @@ class PythonWrapperGenerator(object):
func.add_variant(decl, self.classes, isphantom)
# Add it as global function
g_name = "_".join(classes+[name])
g_name = "_".join(chain(classes, (name, )))
w_classes = []
for i in range(0, len(classes)):
classes_i = classes[:i+1]
@@ -1187,10 +1219,15 @@ class PythonWrapperGenerator(object):
w_classes.append(w_classname)
g_wname = "_".join(w_classes+[name])
func_map = self.namespaces.setdefault(namespace_str, Namespace()).funcs
# Static functions should be called using class names, not like
# module-level functions, so first step is to remove them from
# type hints.
self.typing_stubs_generator.add_ignored_function_name(g_name)
# Exports static function with internal name (backward compatibility)
func = func_map.setdefault(g_name, FuncInfo("", g_name, cname, isconstructor, namespace_str, False))
func.add_variant(decl, self.classes, isphantom)
if g_wname != g_name: # TODO OpenCV 5.0
self.typing_stubs_generator.add_ignored_function_name(g_wname)
wfunc = func_map.setdefault(g_wname, FuncInfo("", g_wname, cname, isconstructor, namespace_str, False))
wfunc.add_variant(decl, self.classes, isphantom)
else:
@@ -1207,7 +1244,6 @@ class PythonWrapperGenerator(object):
if classname and isconstructor:
self.classes[classname].constructor = func
def gen_namespace(self, ns_name):
ns = self.namespaces[ns_name]
wname = normalize_class_name(ns_name)
@@ -1260,6 +1296,7 @@ class PythonWrapperGenerator(object):
self.clear()
self.parser = hdr_parser.CppHeaderParser(generate_umat_decls=True, generate_gpumat_decls=True)
# step 1: scan the headers and build more descriptive maps of classes, consts, functions
for hdr in srcfiles:
decls = self.parser.parse(hdr)
@@ -1355,23 +1392,36 @@ class PythonWrapperGenerator(object):
for decl_idx, name, classinfo in classlist1:
if classinfo.ismap:
continue
def _registerType(classinfo):
if classinfo.decl_idx in published_types:
#print(classinfo.decl_idx, classinfo.name, ' - already published')
return
# If class already registered it means that there is
# a correponding node in the AST. This check is partically
# useful for base classes.
return self.typing_stubs_generator.find_class_node(
classinfo, self.parser.namespaces
)
published_types.add(classinfo.decl_idx)
# Registering a class means creation of the AST node from the
# given class information
class_node = self.typing_stubs_generator.create_class_node(
classinfo, self.parser.namespaces
)
if classinfo.base and classinfo.base in self.classes:
base_classinfo = self.classes[classinfo.base]
#print(classinfo.decl_idx, classinfo.name, ' - request publishing of base type ', base_classinfo.decl_idx, base_classinfo.name)
_registerType(base_classinfo)
# print(classinfo.decl_idx, classinfo.name, ' - request publishing of base type ', base_classinfo.decl_idx, base_classinfo.name)
base_node = _registerType(base_classinfo)
class_node.add_base(base_node)
#print(classinfo.decl_idx, classinfo.name, ' - published!')
# print(classinfo.decl_idx, classinfo.name, ' - published!')
self.code_type_publish.write(classinfo.gen_def(self))
return class_node
_registerType(classinfo)
# step 3: generate the code for all the global functions
for ns_name, ns in sorted(self.namespaces.items()):
if ns_name.split('.')[0] != 'cv':
@@ -1381,6 +1431,10 @@ class PythonWrapperGenerator(object):
continue
code = func.gen_code(self)
self.code_funcs.write(code)
# If function is not ignored - create an AST node for it
if name not in self.typing_stubs_generator.type_hints_ignored_functions:
self.typing_stubs_generator.create_function_node(func)
self.gen_namespace(ns_name)
self.code_ns_init.write('CVPY_MODULE("{}", {});\n'.format(ns_name[2:], normalize_class_name(ns_name)))
@@ -1396,6 +1450,10 @@ class PythonWrapperGenerator(object):
for name, constinfo in constlist:
self.gen_const_reg(constinfo)
# All symbols are collected and AST is reconstructed, generating
# typing stubs...
self.typing_stubs_generator.generate(output_path)
# That's it. Now save all the files
self.save(output_path, "pyopencv_generated_include.h", self.code_include)
self.save(output_path, "pyopencv_generated_funcs.h", self.code_funcs)
@@ -1406,6 +1464,7 @@ class PythonWrapperGenerator(object):
self.save(output_path, "pyopencv_generated_modules_content.h", self.code_ns_reg)
self.save_json(output_path, "pyopencv_signatures.json", self.py_signatures)
if __name__ == "__main__":
srcfiles = hdr_parser.opencv_hdr_list
dstdir = "/Users/vp/tmp"