From 65487946cc4ab121dd8a56462dfce7b64e0377cc Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 24 May 2023 13:29:18 +0300 Subject: [PATCH 001/105] Added final constrants check to solveLP to filter out flating-point numeric issues. --- modules/core/include/opencv2/core/optim.hpp | 7 ++++++- modules/core/src/lpsolver.cpp | 19 +++++++++++++++++-- modules/core/test/test_lpsolver.cpp | 14 ++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/modules/core/include/opencv2/core/optim.hpp b/modules/core/include/opencv2/core/optim.hpp index 5a0940037d..f659d00a72 100644 --- a/modules/core/include/opencv2/core/optim.hpp +++ b/modules/core/include/opencv2/core/optim.hpp @@ -256,6 +256,7 @@ public: //! return codes for cv::solveLP() function enum SolveLPResult { + SOLVELP_LOST = -3, //!< problem is feasible, but solver lost solution due to floating-point arithmetic errors SOLVELP_UNBOUNDED = -2, //!< problem is unbounded (target function can achieve arbitrary high values) SOLVELP_UNFEASIBLE = -1, //!< problem is unfeasible (there are no points that satisfy all the constraints imposed) SOLVELP_SINGLE = 0, //!< there is only one maximum for target function @@ -291,9 +292,13 @@ in the latter case it is understood to correspond to \f$c^T\f$. and the remaining to \f$A\f$. It should contain 32- or 64-bit floating point numbers. @param z The solution will be returned here as a column-vector - it corresponds to \f$c\f$ in the formulation above. It will contain 64-bit floating point numbers. +@param constr_eps allowed numeric disparity for constraints @return One of cv::SolveLPResult */ -CV_EXPORTS_W int solveLP(const Mat& Func, const Mat& Constr, Mat& z); +CV_EXPORTS_W int solveLP(const Mat& Func, const Mat& Constr, Mat& z, double constr_eps); + +/** @overload */ +CV_EXPORTS int solveLP(const Mat& Func, const Mat& Constr, Mat& z); //! @} diff --git a/modules/core/src/lpsolver.cpp b/modules/core/src/lpsolver.cpp index 1a1307d5b5..a00b3191b6 100644 --- a/modules/core/src/lpsolver.cpp +++ b/modules/core/src/lpsolver.cpp @@ -90,7 +90,7 @@ static void swap_columns(Mat_& A,int col1,int col2); #define SWAP(type,a,b) {type tmp=(a);(a)=(b);(b)=tmp;} //return codes:-2 (no_sol - unbdd),-1(no_sol - unfsbl), 0(single_sol), 1(multiple_sol=>least_l2_norm) -int solveLP(const Mat& Func, const Mat& Constr, Mat& z){ +int solveLP(const Mat& Func, const Mat& Constr, Mat& z, double constr_eps){ dprintf(("call to solveLP\n")); //sanity check (size, type, no. of channels) @@ -140,9 +140,24 @@ int solveLP(const Mat& Func, const Mat& Constr, Mat& z){ } } + //check constraints feasibility + Mat prod = Constr(Rect(0, 0, Constr.cols - 1, Constr.rows)) * z; + Mat constr_check = Constr.col(Constr.cols - 1) - prod; + double min_value = 0.0; + minMaxIdx(constr_check, &min_value); + if (min_value < -constr_eps) + { + return SOLVELP_LOST; + } + return res; } +CV_EXPORTS_W int solveLP(const Mat& Func, const Mat& Constr, Mat& z) +{ + return solveLP(Func, Constr, z, 1e-12); +} + static int initialize_simplex(Mat_& c, Mat_& b,double& v,vector& N,vector& B,vector& indexToRow){ N.resize(c.cols); N[0]=0; @@ -255,7 +270,7 @@ static int inner_simplex(Mat_& c, Mat_& b,double& v,vector& dprintf(("iteration #%d\n",count)); count++; - static MatIterator_ pos_ptr; + MatIterator_ pos_ptr; int e=-1,pos_ctr=0,min_var=INT_MAX; bool all_nonzero=true; for(pos_ptr=c.begin();pos_ptr!=c.end();pos_ptr++,pos_ctr++){ diff --git a/modules/core/test/test_lpsolver.cpp b/modules/core/test/test_lpsolver.cpp index 97f87f4c2f..a92782405e 100644 --- a/modules/core/test/test_lpsolver.cpp +++ b/modules/core/test/test_lpsolver.cpp @@ -151,4 +151,18 @@ TEST(Core_LPSolver, issue_12337) //need to update interface: EXPECT_ANY_THROW(Mat1b z_8u; cv::solveLP(A, B, z_8u)); } +// NOTE: Test parameters found experimentally to get numerically inaccurate result. +// The test behaviour may change after algorithm tuning and may removed. +TEST(Core_LPSolver, issue_12343) +{ + Mat A = (cv::Mat_(4, 1) << 3., 3., 3., 4.); + Mat B = (cv::Mat_(4, 5) << 0., 1., 4., 4., 3., + 3., 1., 2., 2., 3., + 4., 4., 0., 1., 4., + 4., 0., 4., 1., 4.); + Mat z; + int result = cv::solveLP(A, B, z); + EXPECT_EQ(SOLVELP_LOST, result); +} + }} // namespace From cbda161c3928fd9a5ee0540a86c909f2ac542ae3 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 24 May 2023 16:16:14 +0300 Subject: [PATCH 002/105] Fixed FPS computation on some videos for FFmpeg backend. --- modules/videoio/src/cap_ffmpeg_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 66388bcb87..b0b81c6fd0 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -1537,7 +1537,7 @@ int64_t CvCapture_FFMPEG::get_bitrate() const double CvCapture_FFMPEG::get_fps() const { -#if 0 && LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(55, 1, 100) && LIBAVFORMAT_VERSION_MICRO >= 100 +#if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(55, 1, 100) && LIBAVFORMAT_VERSION_MICRO >= 100 double fps = r2d(av_guess_frame_rate(ic, ic->streams[video_stream], NULL)); #else #if LIBAVCODEC_BUILD >= CALC_FFMPEG_VERSION(54, 1, 0) From 66f86e898cd920988d42322168945f2d76175bba Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Fri, 2 Jun 2023 10:33:24 +0300 Subject: [PATCH 003/105] Fixed potential buffer overflow of user file name in create_samples_app --- apps/createsamples/utility.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/createsamples/utility.cpp b/apps/createsamples/utility.cpp index 5176f14836..e94792b1e4 100644 --- a/apps/createsamples/utility.cpp +++ b/apps/createsamples/utility.cpp @@ -70,7 +70,7 @@ using namespace cv; static int icvMkDir( const char* filename ) { - char path[PATH_MAX]; + char path[PATH_MAX+1]; char* p; int pos; @@ -83,7 +83,8 @@ static int icvMkDir( const char* filename ) mode = 0755; #endif /* _WIN32 */ - strcpy( path, filename ); + path[0] = '\0'; + strncat( path, filename, PATH_MAX ); p = path; for( ; ; ) From e1ce2146f52bf0e0b9822b321aacbd6217b3bc26 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Fri, 9 Jun 2023 09:21:55 +0000 Subject: [PATCH 004/105] build(ios): disable workaround for CMake 3.25.1+ --- .../cmake/Toolchains/common-ios-toolchain.cmake | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/platforms/ios/cmake/Toolchains/common-ios-toolchain.cmake b/platforms/ios/cmake/Toolchains/common-ios-toolchain.cmake index 735c75f1f3..3325665f91 100644 --- a/platforms/ios/cmake/Toolchains/common-ios-toolchain.cmake +++ b/platforms/ios/cmake/Toolchains/common-ios-toolchain.cmake @@ -98,8 +98,17 @@ if(NOT DEFINED IPHONEOS_DEPLOYMENT_TARGET) endif() if(NOT __IN_TRY_COMPILE) - set(_xcodebuild_wrapper "${CMAKE_BINARY_DIR}/xcodebuild_wrapper") - if(NOT EXISTS "${_xcodebuild_wrapper}") + set(_xcodebuild_wrapper "") + if(NOT (CMAKE_VERSION VERSION_LESS "3.25.1")) # >= 3.25.1 + # no workaround is required (#23156) + elseif(NOT (CMAKE_VERSION VERSION_LESS "3.25.0")) # >= 3.25.0 < 3.25.1 + if(NOT OPENCV_SKIP_MESSAGE_ISSUE_23156) + message(FATAL_ERROR "OpenCV: Please upgrade CMake to 3.25.1+. Current CMake version is ${CMAKE_VERSION}. Details: https://github.com/opencv/opencv/issues/23156") + endif() + else() # < 3.25.0, apply workaround from #13912 + set(_xcodebuild_wrapper "${CMAKE_BINARY_DIR}/xcodebuild_wrapper") + endif() + if(_xcodebuild_wrapper AND NOT EXISTS "${_xcodebuild_wrapper}") set(_xcodebuild_wrapper_tmp "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/xcodebuild_wrapper") if(NOT DEFINED CMAKE_MAKE_PROGRAM) # empty since CMake 3.10 find_program(XCODEBUILD_PATH "xcodebuild") @@ -119,7 +128,9 @@ if(NOT __IN_TRY_COMPILE) configure_file("${CMAKE_CURRENT_LIST_DIR}/xcodebuild_wrapper.in" "${_xcodebuild_wrapper_tmp}" @ONLY) file(COPY "${_xcodebuild_wrapper_tmp}" DESTINATION ${CMAKE_BINARY_DIR} FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) endif() - set(CMAKE_MAKE_PROGRAM "${_xcodebuild_wrapper}" CACHE INTERNAL "" FORCE) + if(_xcodebuild_wrapper) + set(CMAKE_MAKE_PROGRAM "${_xcodebuild_wrapper}" CACHE INTERNAL "" FORCE) + endif() endif() # Standard settings From 5859a531e5185248ab7f4d7ab2cc071f39a6e8c9 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Wed, 14 Jun 2023 21:24:05 +0300 Subject: [PATCH 005/105] feat: manual refinement for Python API definition Mark `resize` and `calcHist` arguments as optional regardless of their C++ API optionality --- .../typing_stubs_generation/api_refinement.py | 44 +++++++++++++++++++ .../src2/typing_stubs_generation/ast_utils.py | 25 ++++++++--- .../typing_stubs_generation/generation.py | 13 ++++++ .../nodes/function_node.py | 30 ++++++++++--- modules/python/src2/typing_stubs_generator.py | 6 ++- 5 files changed, 104 insertions(+), 14 deletions(-) create mode 100644 modules/python/src2/typing_stubs_generation/api_refinement.py diff --git a/modules/python/src2/typing_stubs_generation/api_refinement.py b/modules/python/src2/typing_stubs_generation/api_refinement.py new file mode 100644 index 0000000000..a3468c1567 --- /dev/null +++ b/modules/python/src2/typing_stubs_generation/api_refinement.py @@ -0,0 +1,44 @@ +__all__ = [ + "apply_manual_api_refinement" +] + +from typing import Sequence, Callable +from .nodes import NamespaceNode, FunctionNode, OptionalTypeNode +from .ast_utils import find_function_node, SymbolName + + +def apply_manual_api_refinement(root: NamespaceNode) -> None: + for symbol_name, refine_symbol in NODES_TO_REFINE.items(): + refine_symbol(root, symbol_name) + + +def make_optional_arg(arg_name: str) -> Callable[[NamespaceNode, SymbolName], None]: + def _make_optional_arg(root_node: NamespaceNode, + function_symbol_name: SymbolName) -> None: + function = find_function_node(root_node, function_symbol_name) + for overload in function.overloads: + arg_idx = _find_argument_index(overload.arguments, arg_name) + # Avoid multiplying optional qualification + if isinstance(overload.arguments[arg_idx].type_node, OptionalTypeNode): + continue + + overload.arguments[arg_idx].type_node = OptionalTypeNode( + overload.arguments[arg_idx].type_node + ) + + return _make_optional_arg + + +def _find_argument_index(arguments: Sequence[FunctionNode.Arg], name: str) -> int: + for i, arg in enumerate(arguments): + if arg.name == name: + return i + raise RuntimeError( + f"Failed to find argument with name: '{name}' in {arguments}" + ) + + +NODES_TO_REFINE = { + SymbolName(("cv", ), (), "resize"): make_optional_arg("dsize"), + SymbolName(("cv", ), (), "calcHist"): make_optional_arg("mask"), +} diff --git a/modules/python/src2/typing_stubs_generation/ast_utils.py b/modules/python/src2/typing_stubs_generation/ast_utils.py index 1f802f6da5..7001e1dd9c 100644 --- a/modules/python/src2/typing_stubs_generation/ast_utils.py +++ b/modules/python/src2/typing_stubs_generation/ast_utils.py @@ -142,13 +142,24 @@ because 'GOpaque' class is not registered yet return scope -def find_class_node(root: NamespaceNode, full_class_name: str, - namespaces: Sequence[str]) -> ClassNode: - symbol_name = SymbolName.parse(full_class_name, namespaces) - scope = find_scope(root, symbol_name) - if symbol_name.name not in scope.classes: - raise SymbolNotFoundError("Can't find {} in its scope".format(symbol_name)) - return scope.classes[symbol_name.name] +def find_class_node(root: NamespaceNode, class_symbol: SymbolName, + create_missing_namespaces: bool = False) -> ClassNode: + scope = find_scope(root, class_symbol, create_missing_namespaces) + if class_symbol.name not in scope.classes: + raise SymbolNotFoundError( + "Can't find {} in its scope".format(class_symbol) + ) + return scope.classes[class_symbol.name] + + +def find_function_node(root: NamespaceNode, function_symbol: SymbolName, + create_missing_namespaces: bool = False) -> FunctionNode: + scope = find_scope(root, function_symbol, create_missing_namespaces) + if function_symbol.name not in scope.functions: + raise SymbolNotFoundError( + "Can't find {} in its scope".format(function_symbol) + ) + return scope.functions[function_symbol.name] def create_function_node_in_scope(scope: Union[NamespaceNode, ClassNode], diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index 018c55414a..43fd82e797 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -9,6 +9,7 @@ import warnings from .ast_utils import get_enclosing_namespace from .predefined_types import PREDEFINED_TYPES +from .api_refinement import apply_manual_api_refinement from .nodes import (ASTNode, NamespaceNode, ClassNode, FunctionNode, EnumerationNode, ConstantNode) @@ -45,6 +46,18 @@ def generate_typing_stubs(root: NamespaceNode, output_path: Path): root (NamespaceNode): Root namespace node of the library AST. output_path (Path): Path to output directory. """ + # Perform special handling for function arguments that has some conventions + # not expressed in their API e.g. optionality of mutually exclusive arguments + # without default values: + # ```cxx + # cv::resize(cv::InputArray src, cv::OutputArray dst, cv::Size dsize, + # double fx = 0.0, double fy = 0.0, int interpolation); + # ``` + # should accept `None` as `dsize`: + # ```python + # cv2.resize(image, dsize=None, fx=0.5, fy=0.5) + # ``` + apply_manual_api_refinement(root) # Most of the time type nodes miss their full name (especially function # arguments and return types), so resolution should start from the narrowest # scope and gradually expanded. diff --git a/modules/python/src2/typing_stubs_generation/nodes/function_node.py b/modules/python/src2/typing_stubs_generation/nodes/function_node.py index a4e4f56561..4ebb90633e 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/function_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/function_node.py @@ -10,10 +10,12 @@ class FunctionNode(ASTNode): This class defines an overload set rather then function itself, because function without overloads is represented as FunctionNode with 1 overload. """ - class Arg(NamedTuple): - name: str - type_node: Optional[TypeNode] = None - default_value: Optional[str] = None + class Arg: + def __init__(self, name: str, type_node: Optional[TypeNode] = None, + default_value: Optional[str] = None) -> None: + self.name = name + self.type_node = type_node + self.default_value = default_value @property def typename(self) -> Optional[str]: @@ -24,8 +26,18 @@ class FunctionNode(ASTNode): return self.type_node.relative_typename(root) return None - class RetType(NamedTuple): - type_node: TypeNode = NoneTypeNode("void") + def __str__(self) -> str: + return ( + f"Arg(name={self.name}, type_node={self.type_node}," + f" default_value={self.default_value})" + ) + + def __repr__(self) -> str: + return str(self) + + class RetType: + def __init__(self, type_node: TypeNode = NoneTypeNode("void")) -> None: + self.type_node = type_node @property def typename(self) -> str: @@ -34,6 +46,12 @@ class FunctionNode(ASTNode): def relative_typename(self, root: str) -> Optional[str]: return self.type_node.relative_typename(root) + def __str__(self) -> str: + return f"RetType(type_node={self.type_node})" + + def __repr__(self) -> str: + return str(self) + class Overload(NamedTuple): arguments: Sequence["FunctionNode.Arg"] = () return_type: Optional["FunctionNode.RetType"] = None diff --git a/modules/python/src2/typing_stubs_generator.py b/modules/python/src2/typing_stubs_generator.py index fa0b2b7a28..2ab19bfdf8 100644 --- a/modules/python/src2/typing_stubs_generator.py +++ b/modules/python/src2/typing_stubs_generator.py @@ -120,7 +120,11 @@ if sys.version_info >= (3, 6): @failures_wrapper.wrap_exceptions_as_warnings(ret_type_on_failure=ClassNodeStub) def find_class_node(self, class_info, namespaces): # type: (Any, Sequence[str]) -> ClassNode - return find_class_node(self.cv_root, class_info.full_original_name, namespaces) + return find_class_node( + self.cv_root, + SymbolName.parse(class_info.full_original_name, namespaces), + create_missing_namespaces=True + ) @failures_wrapper.wrap_exceptions_as_warnings(ret_type_on_failure=ClassNodeStub) def create_class_node(self, class_info, namespaces): From 1acbeb217bca6d7f40edce3094259101ff29690d Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Thu, 15 Jun 2023 16:32:48 +0300 Subject: [PATCH 006/105] feat: re-export symbols to cv2 level - Re-export native submodules of cv2 package level. - Re-export manually registered symbols like cv2.mat_wrapper.Mat --- .../typing_stubs_generation/generation.py | 52 ++++++++++++++++++- .../nodes/namespace_node.py | 22 +++++--- 2 files changed, 67 insertions(+), 7 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index 018c55414a..5c32f43ff9 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -70,10 +70,11 @@ def generate_typing_stubs(root: NamespaceNode, output_path: Path): # checked and at least 1 node is still unresolved. root.resolve_type_nodes() _generate_typing_module(root, output_path) + _populate_reexported_symbols(root) _generate_typing_stubs(root, output_path) -def _generate_typing_stubs(root: NamespaceNode, output_path: Path): +def _generate_typing_stubs(root: NamespaceNode, output_path: Path) -> None: output_path = Path(output_path) / root.export_name output_path.mkdir(parents=True, exist_ok=True) @@ -85,6 +86,8 @@ def _generate_typing_stubs(root: NamespaceNode, output_path: Path): # Write required imports at the top of file _write_required_imports(required_imports, output_stream) + _write_reexported_symbols_section(root, output_stream) + # Write constants section, because constants don't impose any dependencies _generate_section_stub(StubSection("# Constants", ConstantNode), root, output_stream, 0) @@ -582,6 +585,53 @@ def _collect_required_imports(root: NamespaceNode) -> Set[str]: return required_imports +def _populate_reexported_symbols(root: NamespaceNode) -> None: + # Re-export all submodules to allow referencing symbols in submodules + # without submodule import. Example: + # `cv2.aruco.ArucoDetector` should be accessible without `import cv2.aruco` + for submodule in root.namespaces.values(): + root.reexported_submodules.append(submodule.export_name) + + # Special cases, symbols defined in possible pure Python submodules should be + root.reexported_submodules_symbols["mat_wrapper"].append("Mat") + +def _write_reexported_symbols_section(module: NamespaceNode, output_stream: StringIO) -> None: + """Write re-export section for the given module. + + Re-export statements have from `from module_name import smth as smth`. + Example: + ```python + from cv2 import aruco as aruco + from cv2 import cuda as cuda + from cv2 import ml as ml + from cv2.mat_wrapper import Mat as Mat + ``` + + Args: + module (NamespaceNode): Module with re-exported symbols. + output_stream (StringIO): Output stream for re-export statements. + """ + + parent_name = module.full_export_name + for submodule in sorted(module.reexported_submodules): + output_stream.write( + "from {0} import {1} as {1}\n".format(parent_name, submodule) + ) + + for submodule, symbols in sorted(module.reexported_submodules_symbols.items(), + key=lambda kv: kv[0]): + for symbol in symbols: + output_stream.write( + "from {0}.{1} import {2} as {2}\n".format( + parent_name, submodule, symbol + ) + ) + + if len(module.reexported_submodules) or \ + len(module.reexported_submodules_symbols): + output_stream.write("\n\n") + + def _write_required_imports(required_imports: Collection[str], output_stream: StringIO) -> None: """Writes all entries of `required_imports` to the `output_stream`. diff --git a/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py b/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py index 2206f0e586..8d0d04b5a5 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py @@ -1,14 +1,13 @@ -from typing import Type, Iterable, Sequence, Tuple, Optional, Dict import itertools import weakref - -from .node import ASTNode, ASTNodeType +from collections import defaultdict +from typing import Dict, List, Optional, Sequence, Tuple, Type from .class_node import ClassNode, ClassProperty -from .function_node import FunctionNode -from .enumeration_node import EnumerationNode from .constant_node import ConstantNode - +from .enumeration_node import EnumerationNode +from .function_node import FunctionNode +from .node import ASTNode, ASTNodeType from .type_node import TypeResolutionError @@ -18,6 +17,17 @@ class NamespaceNode(ASTNode): NamespaceNode can have other namespaces, classes, functions, enumerations and global constants as its children nodes. """ + def __init__(self, name: str, parent: Optional[ASTNode] = None, + export_name: Optional[str] = None) -> None: + super().__init__(name, parent, export_name) + self.reexported_submodules: List[str] = [] + """List of reexported submodules""" + + self.reexported_submodules_symbols: Dict[str, List[str]] = defaultdict(list) + """Mapping between submodules export names and their symbols re-exported + in this module""" + + @property def node_type(self) -> ASTNodeType: return ASTNodeType.Namespace From e625b32841bd5e2a2db281bd1730dfe3af887959 Mon Sep 17 00:00:00 2001 From: dizcza Date: Thu, 15 Jun 2023 19:30:40 +0300 Subject: [PATCH 007/105] [opencv 3.x] back-ported tbb support ubuntu 22.04 --- 3rdparty/tbb/CMakeLists.txt | 2 +- cmake/OpenCVDetectTBB.cmake | 9 +++++---- cmake/OpenCVFindMKL.cmake | 9 ++------- modules/core/src/parallel.cpp | 1 - 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/3rdparty/tbb/CMakeLists.txt b/3rdparty/tbb/CMakeLists.txt index a085b0f3ca..50f3e6ccf1 100644 --- a/3rdparty/tbb/CMakeLists.txt +++ b/3rdparty/tbb/CMakeLists.txt @@ -170,4 +170,4 @@ ocv_install_target(tbb EXPORT OpenCVModules ocv_install_3rdparty_licenses(tbb "${tbb_src_dir}/LICENSE" "${tbb_src_dir}/README") -ocv_tbb_read_version("${tbb_src_dir}/include") +ocv_tbb_read_version("${tbb_src_dir}/include" tbb) diff --git a/cmake/OpenCVDetectTBB.cmake b/cmake/OpenCVDetectTBB.cmake index 38137f44f0..1154bb1d0f 100644 --- a/cmake/OpenCVDetectTBB.cmake +++ b/cmake/OpenCVDetectTBB.cmake @@ -19,7 +19,7 @@ # - "tbb" target exists and added to OPENCV_LINKER_LIBS function(ocv_tbb_cmake_guess _found) - find_package(TBB QUIET COMPONENTS tbb PATHS "$ENV{TBBROOT}/cmake") + find_package(TBB QUIET COMPONENTS tbb PATHS "$ENV{TBBROOT}/cmake" "$ENV{TBBROOT}/lib/cmake/tbb") if(TBB_FOUND) if(NOT TARGET TBB::tbb) message(WARNING "No TBB::tbb target found!") @@ -28,11 +28,11 @@ function(ocv_tbb_cmake_guess _found) get_target_property(_lib TBB::tbb IMPORTED_LOCATION_RELEASE) message(STATUS "Found TBB (cmake): ${_lib}") get_target_property(_inc TBB::tbb INTERFACE_INCLUDE_DIRECTORIES) - ocv_tbb_read_version("${_inc}") add_library(tbb INTERFACE IMPORTED) set_target_properties(tbb PROPERTIES INTERFACE_LINK_LIBRARIES TBB::tbb ) + ocv_tbb_read_version("${_inc}" tbb) set(${_found} TRUE PARENT_SCOPE) endif() endfunction() @@ -66,7 +66,6 @@ function(ocv_tbb_env_guess _found) find_library(TBB_ENV_LIB_DEBUG NAMES "tbb_debug") if (TBB_ENV_INCLUDE AND (TBB_ENV_LIB OR TBB_ENV_LIB_DEBUG)) ocv_tbb_env_verify() - ocv_tbb_read_version("${TBB_ENV_INCLUDE}") add_library(tbb UNKNOWN IMPORTED) set_target_properties(tbb PROPERTIES IMPORTED_LOCATION "${TBB_ENV_LIB}" @@ -82,12 +81,14 @@ function(ocv_tbb_env_guess _found) get_filename_component(_dir "${TBB_ENV_LIB}" DIRECTORY) set_target_properties(tbb PROPERTIES INTERFACE_LINK_LIBRARIES "-L${_dir}") endif() + ocv_tbb_read_version("${TBB_ENV_INCLUDE}" tbb) message(STATUS "Found TBB (env): ${TBB_ENV_LIB}") set(${_found} TRUE PARENT_SCOPE) endif() endfunction() -function(ocv_tbb_read_version _path) +function(ocv_tbb_read_version _path _tgt) + find_file(TBB_VER_FILE oneapi/tbb/version.h "${_path}" NO_DEFAULT_PATH CMAKE_FIND_ROOT_PATH_BOTH) find_file(TBB_VER_FILE tbb/tbb_stddef.h "${_path}" NO_DEFAULT_PATH CMAKE_FIND_ROOT_PATH_BOTH) ocv_parse_header("${TBB_VER_FILE}" TBB_VERSION_LINES TBB_VERSION_MAJOR TBB_VERSION_MINOR TBB_INTERFACE_VERSION CACHE) endfunction() diff --git a/cmake/OpenCVFindMKL.cmake b/cmake/OpenCVFindMKL.cmake index 0638e89e2b..1b6d47a271 100644 --- a/cmake/OpenCVFindMKL.cmake +++ b/cmake/OpenCVFindMKL.cmake @@ -118,16 +118,10 @@ if(MKL_USE_SINGLE_DYNAMIC_LIBRARY AND NOT (MKL_VERSION_STR VERSION_LESS "10.3.0" elseif(NOT (MKL_VERSION_STR VERSION_LESS "11.3.0")) - foreach(MKL_ARCH ${MKL_ARCH_LIST}) - list(APPEND mkl_lib_find_paths - ${MKL_ROOT_DIR}/../tbb/lib/${MKL_ARCH} - ) - endforeach() - set(mkl_lib_list "mkl_intel_${MKL_ARCH_SUFFIX}") if(MKL_WITH_TBB) - list(APPEND mkl_lib_list mkl_tbb_thread tbb) + list(APPEND mkl_lib_list mkl_tbb_thread) elseif(MKL_WITH_OPENMP) if(MSVC) list(APPEND mkl_lib_list mkl_intel_thread libiomp5md) @@ -155,6 +149,7 @@ if(NOT MKL_LIBRARIES) endif() list(APPEND MKL_LIBRARIES ${${lib_var_name}}) endforeach() + list(APPEND MKL_LIBRARIES ${OPENCV_EXTRA_MKL_LIBRARIES}) endif() message(STATUS "Found MKL ${MKL_VERSION_STR} at: ${MKL_ROOT_DIR}") diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index efecefc4b4..dc52820e71 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -101,7 +101,6 @@ #endif #include "tbb/tbb.h" #include "tbb/task.h" - #include "tbb/tbb_stddef.h" #if TBB_INTERFACE_VERSION >= 8000 #include "tbb/task_arena.h" #endif From a3b6a5b606f7b143c0ff201791b20fd3f0af055a Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Thu, 15 Jun 2023 20:21:08 +0300 Subject: [PATCH 008/105] fix: typing module enums references Enum names exist only during type checking. During runtime they should be denoted as named integral types --- .../src2/typing_stubs_generation/ast_utils.py | 33 +++++++++- .../typing_stubs_generation/generation.py | 61 ++++++++++++++++--- .../typing_stubs_generation/nodes/__init__.py | 2 +- .../typing_stubs_generation/nodes/node.py | 2 +- .../nodes/type_node.py | 4 ++ modules/python/src2/typing_stubs_generator.py | 7 ++- 6 files changed, 94 insertions(+), 15 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/ast_utils.py b/modules/python/src2/typing_stubs_generation/ast_utils.py index 1f802f6da5..92eb87858d 100644 --- a/modules/python/src2/typing_stubs_generation/ast_utils.py +++ b/modules/python/src2/typing_stubs_generation/ast_utils.py @@ -1,4 +1,5 @@ -from typing import NamedTuple, Sequence, Tuple, Union, List, Dict +from typing import (NamedTuple, Sequence, Tuple, Union, List, + Dict, Callable, Optional) import keyword from .nodes import (ASTNode, NamespaceNode, ClassNode, FunctionNode, @@ -323,12 +324,18 @@ def resolve_enum_scopes(root: NamespaceNode, enum_node.parent = scope -def get_enclosing_namespace(node: ASTNode) -> NamespaceNode: +def get_enclosing_namespace( + node: ASTNode, + class_node_callback: Optional[Callable[[ClassNode], None]] = None +) -> NamespaceNode: """Traverses up nodes hierarchy to find closest enclosing namespace of the passed node Args: node (ASTNode): Node to find a namespace for. + class_node_callback (Optional[Callable[[ClassNode], None]]): Optional + callable object invoked for each traversed class node in bottom-up + order. Defaults: None. Returns: NamespaceNode: Closest enclosing namespace of the provided node. @@ -360,10 +367,32 @@ def get_enclosing_namespace(node: ASTNode) -> NamespaceNode: "Can't find enclosing namespace for '{}' known as: '{}'".format( node.full_export_name, node.native_name ) + if class_node_callback: + class_node_callback(parent_node) parent_node = parent_node.parent return parent_node +def get_enum_module_and_export_name(enum_node: EnumerationNode) -> Tuple[str, str]: + """Get export name of the enum node with its module name. + + Note: Enumeration export names are prefixed with enclosing class names. + + Args: + enum_node (EnumerationNode): Enumeration node to construct name for. + + Returns: + Tuple[str, str]: a pair of enum export name and its full module name. + """ + def update_full_export_name(class_node: ClassNode) -> None: + nonlocal enum_export_name + enum_export_name = class_node.export_name + "_" + enum_export_name + + enum_export_name = enum_node.export_name + namespace_node = get_enclosing_namespace(enum_node, update_full_export_name) + return enum_export_name, namespace_node.full_export_name + + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index 4330683774..a7254f9da1 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -6,15 +6,15 @@ from typing import (Generator, Type, Callable, NamedTuple, Union, Set, Dict, Collection) import warnings -from .ast_utils import get_enclosing_namespace +from .ast_utils import get_enclosing_namespace, get_enum_module_and_export_name from .predefined_types import PREDEFINED_TYPES -from .nodes import (ASTNode, NamespaceNode, ClassNode, FunctionNode, +from .nodes import (ASTNode, ASTNodeType, NamespaceNode, ClassNode, FunctionNode, EnumerationNode, ConstantNode) from .nodes.type_node import (TypeNode, AliasTypeNode, AliasRefTypeNode, - AggregatedTypeNode) + AggregatedTypeNode, ASTNodeTypeNode) def generate_typing_stubs(root: NamespaceNode, output_path: Path): @@ -616,29 +616,67 @@ def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None: for item in filter(lambda i: isinstance(i, AliasRefTypeNode), type_node): register_alias(PREDEFINED_TYPES[item.ctype_name]) # type: ignore + def create_alias_for_enum_node(enum_node: ASTNode) -> AliasTypeNode: + """Create int alias corresponding to the given enum node. + + Args: + enum_node (ASTNodeTypeNode): Enumeration node to create int alias for. + + Returns: + AliasTypeNode: int alias node with same export name as enum. + """ + assert enum_node.node_type == ASTNodeType.Enumeration, \ + f"{enum_node} has wrong node type. Expected type: Enumeration." + + enum_export_name, enum_module_name = get_enum_module_and_export_name( + enum_node + ) + enum_full_export_name = f"{enum_module_name}.{enum_export_name}" + alias_node = AliasTypeNode.int_(enum_full_export_name, + enum_export_name) + type_checking_time_definitions.add(alias_node) + return alias_node + def register_alias(alias_node: AliasTypeNode) -> None: typename = alias_node.typename # Check if alias is already registered if typename in aliases: return + + # Collect required imports for alias definition + for required_import in alias_node.required_definition_imports: + required_imports.add(required_import) + if isinstance(alias_node.value, AggregatedTypeNode): # Check if collection contains a link to another alias register_alias_links_from_aggregated_type(alias_node.value) + # Remove references to alias nodes + for i, item in enumerate(alias_node.value.items): + # Process enumerations only + if not isinstance(item, ASTNodeTypeNode) or item.ast_node is None: + continue + if item.ast_node.node_type != ASTNodeType.Enumeration: + continue + alias_node.value.items[i] = create_alias_for_enum_node(item.ast_node) + + if isinstance(alias_node.value, ASTNodeTypeNode) \ + and alias_node.value.ast_node == ASTNodeType.Enumeration: + alias_node.value = create_alias_for_enum_node(alias_node.ast_node) + # Strip module prefix from aliased types aliases[typename] = alias_node.value.full_typename.replace( root.export_name + ".typing.", "" ) if alias_node.doc is not None: aliases[typename] += f'\n"""{alias_node.doc}"""' - for required_import in alias_node.required_definition_imports: - required_imports.add(required_import) output_path = Path(output_path) / root.export_name / "typing" output_path.mkdir(parents=True, exist_ok=True) required_imports: Set[str] = set() aliases: Dict[str, str] = {} + type_checking_time_definitions: Set[AliasTypeNode] = set() # Resolve each node and register aliases TypeNode.compatible_to_runtime_usage = True @@ -655,11 +693,16 @@ def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None: _write_required_imports(required_imports, output_stream) + # Add type checking time definitions as generated __init__.py content + for alias in type_checking_time_definitions: + output_stream.write("if typing.TYPE_CHECKING:\n ") + output_stream.write(f"{alias.typename} = {alias.ctype_name}\nelse:\n") + output_stream.write(f" {alias.typename} = {alias.value.ctype_name}\n") + if type_checking_time_definitions: + output_stream.write("\n\n") + for alias_name, alias_type in aliases.items(): - output_stream.write(alias_name) - output_stream.write(" = ") - output_stream.write(alias_type) - output_stream.write("\n") + output_stream.write(f"{alias_name} = {alias_type}\n") TypeNode.compatible_to_runtime_usage = False (output_path / "__init__.py").write_text(output_stream.getvalue()) diff --git a/modules/python/src2/typing_stubs_generation/nodes/__init__.py b/modules/python/src2/typing_stubs_generation/nodes/__init__.py index fafcd9bc2b..0ee1df93d9 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/__init__.py +++ b/modules/python/src2/typing_stubs_generation/nodes/__init__.py @@ -1,4 +1,4 @@ -from .node import ASTNode +from .node import ASTNode, ASTNodeType from .namespace_node import NamespaceNode from .class_node import ClassNode, ClassProperty from .function_node import FunctionNode diff --git a/modules/python/src2/typing_stubs_generation/nodes/node.py b/modules/python/src2/typing_stubs_generation/nodes/node.py index cba74c8977..126f668018 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/node.py @@ -1,7 +1,7 @@ import abc import enum import itertools -from typing import (Iterator, Type, TypeVar, Iterable, Dict, +from typing import (Iterator, Type, TypeVar, Dict, Optional, Tuple, DefaultDict) from collections import defaultdict diff --git a/modules/python/src2/typing_stubs_generation/nodes/type_node.py b/modules/python/src2/typing_stubs_generation/nodes/type_node.py index a21f983b20..f7018938c1 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/type_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/type_node.py @@ -417,6 +417,10 @@ class ASTNodeTypeNode(TypeNode): self._module_name = module_name self._ast_node: Optional[weakref.ProxyType[ASTNode]] = None + @property + def ast_node(self): + return self._ast_node + @property def typename(self) -> str: if self._ast_node is None: diff --git a/modules/python/src2/typing_stubs_generator.py b/modules/python/src2/typing_stubs_generator.py index fa0b2b7a28..ef801128bc 100644 --- a/modules/python/src2/typing_stubs_generator.py +++ b/modules/python/src2/typing_stubs_generator.py @@ -12,6 +12,7 @@ if sys.version_info >= (3, 6): from contextlib import contextmanager from typing import Dict, Set, Any, Sequence, Generator, Union + import traceback from pathlib import Path @@ -46,10 +47,12 @@ if sys.version_info >= (3, 6): try: ret_type = func(*args, **kwargs) - except Exception as e: + except Exception: self.has_failure = True warnings.warn( - 'Typing stubs generation has failed. Reason: {}'.format(e) + "Typing stubs generation has failed.\n{}".format( + traceback.format_exc() + ) ) if ret_type_on_failure is None: return None From 8762c37c22e37a5102778dc5524e84e27c017f5a Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 15 Jun 2023 21:29:18 +0200 Subject: [PATCH 009/105] solve issue 23808 --- modules/videoio/src/cap_images.cpp | 32 ++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/modules/videoio/src/cap_images.cpp b/modules/videoio/src/cap_images.cpp index b506dd1c06..21b6d5c0da 100644 --- a/modules/videoio/src/cap_images.cpp +++ b/modules/videoio/src/cap_images.cpp @@ -113,7 +113,16 @@ void CvCapture_Images::close() bool CvCapture_Images::grabFrame() { - cv::String filename = cv::format(filename_pattern.c_str(), (int)(firstframe + currentframe)); + cv::String filename; + if (length == 1) + if (currentframe < length) + filename = filename_pattern; + else + { + return false; + } + else + filename = cv::format(filename_pattern.c_str(), (int)(firstframe + currentframe)); CV_Assert(!filename.empty()); if (grabbedInOpen) @@ -250,6 +259,7 @@ std::string icvExtractPattern(const std::string& filename, unsigned *offset) if (pos == len) { + return ""; CV_Error_(Error::StsBadArg, ("CAP_IMAGES: can't find starting number (in the name of file): %s", filename.c_str())); } @@ -292,7 +302,25 @@ bool CvCapture_Images::open(const std::string& _filename) CV_Assert(!_filename.empty()); filename_pattern = icvExtractPattern(_filename, &offset); - CV_Assert(!filename_pattern.empty()); + if (filename_pattern.empty()) + { + filename_pattern = _filename; + cv::String filename = _filename.c_str(); + if (!utils::fs::exists(filename)) + { + return false; + } + if (!haveImageReader(filename)) + { + CV_LOG_INFO(NULL, "CAP_IMAGES: Stop scanning. Can't read image file: " << filename); + } + length = 1; + // grab frame to enable properties retrieval + bool grabRes = grabFrame(); + grabbedInOpen = true; + currentframe = 0; + return grabRes; + } // determine the length of the sequence for (length = 0; ;) From 69ebecc54f1dbc80e4cf8f0905eb79f6c1022247 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Thu, 15 Jun 2023 23:10:10 +0300 Subject: [PATCH 010/105] feat: add OpenCV error class to cv2/__init__.pyi --- modules/python/src2/typing_stubs_generation/api_refinement.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/python/src2/typing_stubs_generation/api_refinement.py b/modules/python/src2/typing_stubs_generation/api_refinement.py index a3468c1567..f23167f914 100644 --- a/modules/python/src2/typing_stubs_generation/api_refinement.py +++ b/modules/python/src2/typing_stubs_generation/api_refinement.py @@ -8,6 +8,10 @@ from .ast_utils import find_function_node, SymbolName def apply_manual_api_refinement(root: NamespaceNode) -> None: + # Export OpenCV exception class + builtin_exception = root.add_class("Exception") + builtin_exception.is_exported = False + root.add_class("error", (builtin_exception, )) for symbol_name, refine_symbol in NODES_TO_REFINE.items(): refine_symbol(root, symbol_name) From 44881592c382e091329d787edf73cfcc093b4233 Mon Sep 17 00:00:00 2001 From: Maksym Ivashechkin Date: Fri, 16 Jun 2023 08:59:13 +0100 Subject: [PATCH 011/105] Merge pull request #23078 from ivashmak:update_vsac Update USAC #23078 ### Pull Request Readiness Checklist - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/calib3d/include/opencv2/calib3d.hpp | 15 +- modules/calib3d/src/five-point.cpp | 4 +- modules/calib3d/src/fundam.cpp | 8 +- modules/calib3d/src/ptsetreg.cpp | 4 +- modules/calib3d/src/solvepnp.cpp | 17 +- modules/calib3d/src/usac.hpp | 401 +++-- modules/calib3d/src/usac/bundle.cpp | 308 ++++ modules/calib3d/src/usac/degeneracy.cpp | 717 ++++++-- modules/calib3d/src/usac/dls_solver.cpp | 31 +- modules/calib3d/src/usac/essential_solver.cpp | 448 ++--- modules/calib3d/src/usac/estimator.cpp | 97 +- .../calib3d/src/usac/fundamental_solver.cpp | 497 +++-- modules/calib3d/src/usac/gamma_values.cpp | 225 ++- .../calib3d/src/usac/homography_solver.cpp | 559 ++++-- .../calib3d/src/usac/local_optimization.cpp | 547 +++--- modules/calib3d/src/usac/pnp_solver.cpp | 62 +- modules/calib3d/src/usac/quality.cpp | 566 +++--- modules/calib3d/src/usac/ransac_solvers.cpp | 1596 +++++++++++------ modules/calib3d/src/usac/sampler.cpp | 30 +- modules/calib3d/src/usac/termination.cpp | 148 +- modules/calib3d/src/usac/utils.cpp | 514 +++++- modules/calib3d/test/test_usac.cpp | 13 +- 22 files changed, 4493 insertions(+), 2314 deletions(-) create mode 100644 modules/calib3d/src/usac/bundle.cpp diff --git a/modules/calib3d/include/opencv2/calib3d.hpp b/modules/calib3d/include/opencv2/calib3d.hpp index a760d3636e..ec4c275597 100644 --- a/modules/calib3d/include/opencv2/calib3d.hpp +++ b/modules/calib3d/include/opencv2/calib3d.hpp @@ -548,12 +548,13 @@ enum RobotWorldHandEyeCalibrationMethod CALIB_ROBOT_WORLD_HAND_EYE_LI = 1 //!< Simultaneous robot-world and hand-eye calibration using dual-quaternions and kronecker product @cite Li2010SimultaneousRA }; -enum SamplingMethod { SAMPLING_UNIFORM, SAMPLING_PROGRESSIVE_NAPSAC, SAMPLING_NAPSAC, - SAMPLING_PROSAC }; -enum LocalOptimMethod {LOCAL_OPTIM_NULL, LOCAL_OPTIM_INNER_LO, LOCAL_OPTIM_INNER_AND_ITER_LO, - LOCAL_OPTIM_GC, LOCAL_OPTIM_SIGMA}; -enum ScoreMethod {SCORE_METHOD_RANSAC, SCORE_METHOD_MSAC, SCORE_METHOD_MAGSAC, SCORE_METHOD_LMEDS}; -enum NeighborSearchMethod { NEIGH_FLANN_KNN, NEIGH_GRID, NEIGH_FLANN_RADIUS }; +enum SamplingMethod { SAMPLING_UNIFORM=0, SAMPLING_PROGRESSIVE_NAPSAC=1, SAMPLING_NAPSAC=2, + SAMPLING_PROSAC=3 }; +enum LocalOptimMethod {LOCAL_OPTIM_NULL=0, LOCAL_OPTIM_INNER_LO=1, LOCAL_OPTIM_INNER_AND_ITER_LO=2, + LOCAL_OPTIM_GC=3, LOCAL_OPTIM_SIGMA=4}; +enum ScoreMethod {SCORE_METHOD_RANSAC=0, SCORE_METHOD_MSAC=1, SCORE_METHOD_MAGSAC=2, SCORE_METHOD_LMEDS=3}; +enum NeighborSearchMethod { NEIGH_FLANN_KNN=0, NEIGH_GRID=1, NEIGH_FLANN_RADIUS=2 }; +enum PolishingMethod { NONE_POLISHER=0, LSQ_POLISHER=1, MAGSAC=2, COV_POLISHER=3 }; struct CV_EXPORTS_W_SIMPLE UsacParams { // in alphabetical order @@ -569,6 +570,8 @@ struct CV_EXPORTS_W_SIMPLE UsacParams CV_PROP_RW SamplingMethod sampler; CV_PROP_RW ScoreMethod score; CV_PROP_RW double threshold; + CV_PROP_RW PolishingMethod final_polisher; + CV_PROP_RW int final_polisher_iterations; }; /** @brief Converts a rotation matrix to a rotation vector or vice versa. diff --git a/modules/calib3d/src/five-point.cpp b/modules/calib3d/src/five-point.cpp index 9b82dcb11c..b8d0b43edc 100644 --- a/modules/calib3d/src/five-point.cpp +++ b/modules/calib3d/src/five-point.cpp @@ -522,9 +522,9 @@ cv::Mat cv::findEssentialMat( InputArray points1, InputArray points2, InputArray cameraMatrix1, InputArray cameraMatrix2, InputArray dist_coeff1, InputArray dist_coeff2, OutputArray mask, const UsacParams ¶ms) { Ptr model; - usac::setParameters(model, usac::EstimationMethod::Essential, params, mask.needed()); + usac::setParameters(model, usac::EstimationMethod::ESSENTIAL, params, mask.needed()); Ptr ransac_output; - if (usac::run(model, points1, points2, model->getRandomGeneratorState(), + if (usac::run(model, points1, points2, ransac_output, cameraMatrix1, cameraMatrix2, dist_coeff1, dist_coeff2)) { usac::saveMask(mask, ransac_output->getInliersMask()); return ransac_output->getModel(); diff --git a/modules/calib3d/src/fundam.cpp b/modules/calib3d/src/fundam.cpp index f0a7df605b..8e07efa26d 100644 --- a/modules/calib3d/src/fundam.cpp +++ b/modules/calib3d/src/fundam.cpp @@ -451,9 +451,9 @@ cv::Mat cv::findHomography( InputArray _points1, InputArray _points2, cv::Mat cv::findHomography(InputArray srcPoints, InputArray dstPoints, OutputArray mask, const UsacParams ¶ms) { Ptr model; - usac::setParameters(model, usac::EstimationMethod::Homography, params, mask.needed()); + usac::setParameters(model, usac::EstimationMethod::HOMOGRAPHY, params, mask.needed()); Ptr ransac_output; - if (usac::run(model, srcPoints, dstPoints, model->getRandomGeneratorState(), + if (usac::run(model, srcPoints, dstPoints, ransac_output, noArray(), noArray(), noArray(), noArray())) { usac::saveMask(mask, ransac_output->getInliersMask()); return ransac_output->getModel() / ransac_output->getModel().at(2,2); @@ -913,10 +913,10 @@ cv::Mat cv::findFundamentalMat( cv::InputArray points1, cv::InputArray points2, cv::Mat cv::findFundamentalMat( InputArray points1, InputArray points2, OutputArray mask, const UsacParams ¶ms) { Ptr model; - setParameters(model, usac::EstimationMethod::Fundamental, params, mask.needed()); + setParameters(model, usac::EstimationMethod::FUNDAMENTAL, params, mask.needed()); CV_Assert(model); Ptr ransac_output; - if (usac::run(model, points1, points2, model->getRandomGeneratorState(), + if (usac::run(model, points1, points2, ransac_output, noArray(), noArray(), noArray(), noArray())) { usac::saveMask(mask, ransac_output->getInliersMask()); return ransac_output->getModel(); diff --git a/modules/calib3d/src/ptsetreg.cpp b/modules/calib3d/src/ptsetreg.cpp index 5c91fff037..c0132a6832 100644 --- a/modules/calib3d/src/ptsetreg.cpp +++ b/modules/calib3d/src/ptsetreg.cpp @@ -1086,9 +1086,9 @@ Mat estimateAffine2D(InputArray _from, InputArray _to, OutputArray _inliers, Mat estimateAffine2D(InputArray _from, InputArray _to, OutputArray inliers, const UsacParams ¶ms) { Ptr model; - usac::setParameters(model, usac::EstimationMethod::Affine, params, inliers.needed()); + usac::setParameters(model, usac::EstimationMethod::AFFINE, params, inliers.needed()); Ptr ransac_output; - if (usac::run(model, _from, _to, model->getRandomGeneratorState(), + if (usac::run(model, _from, _to, ransac_output, noArray(), noArray(), noArray(), noArray())) { usac::saveMask(inliers, ransac_output->getInliersMask()); return ransac_output->getModel().rowRange(0,2); diff --git a/modules/calib3d/src/solvepnp.cpp b/modules/calib3d/src/solvepnp.cpp index d9e2664877..54e23ca9f8 100644 --- a/modules/calib3d/src/solvepnp.cpp +++ b/modules/calib3d/src/solvepnp.cpp @@ -199,21 +199,6 @@ public: Mat tvec; }; -UsacParams::UsacParams() -{ - confidence = 0.99; - isParallel = false; - loIterations = 5; - loMethod = LocalOptimMethod::LOCAL_OPTIM_INNER_LO; - loSampleSize = 14; - maxIterations = 5000; - neighborsSearch = NeighborSearchMethod::NEIGH_GRID; - randomGeneratorState = 0; - sampler = SamplingMethod::SAMPLING_UNIFORM; - score = ScoreMethod::SCORE_METHOD_MSAC; - threshold = 1.5; -} - bool solvePnPRansac(InputArray _opoints, InputArray _ipoints, InputArray _cameraMatrix, InputArray _distCoeffs, OutputArray _rvec, OutputArray _tvec, bool useExtrinsicGuess, @@ -407,7 +392,7 @@ bool solvePnPRansac( InputArray objectPoints, InputArray imagePoints, usac::setParameters(model_params, cameraMatrix.empty() ? usac::EstimationMethod::P6P : usac::EstimationMethod::P3P, params, inliers.needed()); Ptr ransac_output; - if (usac::run(model_params, imagePoints, objectPoints, model_params->getRandomGeneratorState(), + if (usac::run(model_params, imagePoints, objectPoints, ransac_output, cameraMatrix, noArray(), distCoeffs, noArray())) { if (inliers.needed()) { const auto &inliers_mask = ransac_output->getInliersMask(); diff --git a/modules/calib3d/src/usac.hpp b/modules/calib3d/src/usac.hpp index 2a3e0eb7fc..85b2730e4a 100644 --- a/modules/calib3d/src/usac.hpp +++ b/modules/calib3d/src/usac.hpp @@ -6,10 +6,11 @@ #define OPENCV_USAC_USAC_HPP namespace cv { namespace usac { -enum EstimationMethod { Homography, Fundamental, Fundamental8, Essential, Affine, P3P, P6P}; -enum VerificationMethod { NullVerifier, SprtVerifier }; -enum PolishingMethod { NonePolisher, LSQPolisher }; -enum ErrorMetric {DIST_TO_LINE, SAMPSON_ERR, SGD_ERR, SYMM_REPR_ERR, FORW_REPR_ERR, RERPOJ}; +enum EstimationMethod { HOMOGRAPHY=0, FUNDAMENTAL=1, FUNDAMENTAL8=2, ESSENTIAL=3, AFFINE=4, P3P=5, P6P=6}; +enum VerificationMethod { NULL_VERIFIER=0, SPRT_VERIFIER=1, ASPRT=2 }; +enum ErrorMetric {DIST_TO_LINE=0, SAMPSON_ERR=1, SGD_ERR=2, SYMM_REPR_ERR=3, FORW_REPR_ERR=4, RERPOJ=5}; +enum MethodSolver { GEM_SOLVER=0, SVD_SOLVER=1 }; +enum ModelConfidence {RANDOM=0, NON_RANDOM=1, UNKNOWN=2}; // Abstract Error class class Error : public Algorithm { @@ -19,7 +20,6 @@ public: // returns error of point wih @point_idx w.r.t. model virtual float getError (int point_idx) const = 0; virtual const std::vector &getErrors (const Mat &model) = 0; - virtual Ptr clone () const = 0; }; // Symmetric Reprojection Error for Homography @@ -58,6 +58,11 @@ public: static Ptr create(const Mat &points); }; +class TrifocalTensorReprError : public Error { +public: + static Ptr create(const Mat &points); +}; + // Normalizing transformation of data points class NormTransform : public Algorithm { public: @@ -82,19 +87,22 @@ public: virtual int getSampleSize() const = 0; // return maximum number of possible solutions. virtual int getMaxNumberOfSolutions () const = 0; - virtual Ptr clone () const = 0; }; //-------------------------- HOMOGRAPHY MATRIX ----------------------- -class HomographyMinimalSolver4ptsGEM : public MinimalSolver { +class HomographyMinimalSolver4pts : public MinimalSolver { public: - static Ptr create(const Mat &points_); + static Ptr create(const Mat &points, bool use_ge); +}; +class PnPSVDSolver : public MinimalSolver { +public: + static Ptr create (const Mat &points); }; //-------------------------- FUNDAMENTAL MATRIX ----------------------- class FundamentalMinimalSolver7pts : public MinimalSolver { public: - static Ptr create(const Mat &points_); + static Ptr create(const Mat &points, bool use_ge); }; class FundamentalMinimalSolver8pts : public MinimalSolver { @@ -103,9 +111,9 @@ public: }; //-------------------------- ESSENTIAL MATRIX ----------------------- -class EssentialMinimalSolverStewenius5pts : public MinimalSolver { +class EssentialMinimalSolver5pts : public MinimalSolver { public: - static Ptr create(const Mat &points_); + static Ptr create(const Mat &points, bool use_svd, bool is_nister); }; //-------------------------- PNP ----------------------- @@ -116,7 +124,7 @@ public: class P3PSolver : public MinimalSolver { public: - static Ptr create(const Mat &points_, const Mat &calib_norm_pts, const Matx33d &K); + static Ptr create(const Mat &points_, const Mat &calib_norm_pts, const Mat &K); }; //-------------------------- AFFINE ----------------------- @@ -125,9 +133,20 @@ public: static Ptr create(const Mat &points_); }; +class TrifocalTensorMinimalSolver : public MinimalSolver { +public: + static Ptr create(const Mat &points_); + virtual void getFundamentalMatricesFromTensor (const cv::Mat &tensor, cv::Mat &F21, cv::Mat &F31) = 0; +}; + //////////////////////////////////////// NON MINIMAL SOLVER /////////////////////////////////////// class NonMinimalSolver : public Algorithm { public: + virtual int estimate (const Mat &model, const std::vector &sample, int sample_size, std::vector + &models, const std::vector &weights) const { + CV_UNUSED(model); + return estimate(sample, sample_size, models, weights); + } // Estimate models from non minimal sample. models.size() == number of found solutions virtual int estimate (const std::vector &sample, int sample_size, std::vector &models, const std::vector &weights) const = 0; @@ -135,25 +154,34 @@ public: virtual int getMinimumRequiredSampleSize() const = 0; // return maximum number of possible solutions. virtual int getMaxNumberOfSolutions () const = 0; - virtual Ptr clone () const = 0; + virtual int estimate (const std::vector& mask, std::vector& models, + const std::vector& weights) = 0; + virtual void enforceRankConstraint (bool enforce) = 0; }; //-------------------------- HOMOGRAPHY MATRIX ----------------------- class HomographyNonMinimalSolver : public NonMinimalSolver { public: - static Ptr create(const Mat &points_); + static Ptr create(const Mat &points_, bool use_ge_=false); + static Ptr create(const Mat &points_, const Matx33d &T1, const Matx33d &T2, bool use_ge); }; //-------------------------- FUNDAMENTAL MATRIX ----------------------- -class FundamentalNonMinimalSolver : public NonMinimalSolver { +class EpipolarNonMinimalSolver : public NonMinimalSolver { public: - static Ptr create(const Mat &points_); + static Ptr create(const Mat &points_, bool is_fundamental); + static Ptr create(const Mat &points_, const Matx33d &T1, const Matx33d &T2, bool use_ge); }; //-------------------------- ESSENTIAL MATRIX ----------------------- -class EssentialNonMinimalSolver : public NonMinimalSolver { +class EssentialNonMinimalSolverViaF : public NonMinimalSolver { public: - static Ptr create(const Mat &points_); +static Ptr create(const Mat &points_, const cv::Mat &K1, const Mat &K2); +}; + +class EssentialNonMinimalSolverViaT : public NonMinimalSolver { +public: + static Ptr create(const Mat &points_); }; //-------------------------- PNP ----------------------- @@ -164,16 +192,20 @@ public: class DLSPnP : public NonMinimalSolver { public: - static Ptr create(const Mat &points_, const Mat &calib_norm_pts, const Matx33d &K); + static Ptr create(const Mat &points_, const Mat &calib_norm_pts, const Mat &K); }; //-------------------------- AFFINE ----------------------- class AffineNonMinimalSolver : public NonMinimalSolver { public: - static Ptr create(const Mat &points_); + static Ptr create(const Mat &points, InputArray T1, InputArray T2); +}; + +class LarssonOptimizer : public NonMinimalSolver { +public: + static Ptr create(const Mat &calib_points_, const Matx33d &K1_, const Matx33d &K2_, int max_iters_, bool is_fundamental_); }; -////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////// SCORE /////////////////////////////////////////// class Score { public: @@ -193,24 +225,16 @@ public: } }; -class GammaValues -{ - const double max_range_complete /*= 4.62*/, max_range_gamma /*= 1.52*/; - const int max_size_table /* = 3000 */; - - std::vector gamma_complete, gamma_incomplete, gamma; - - GammaValues(); // use getSingleton() - +class GammaValues : public Algorithm { public: - static const GammaValues& getSingleton(); - - const std::vector& getCompleteGammaValues() const; - const std::vector& getIncompleteGammaValues() const; - const std::vector& getGammaValues() const; - double getScaleOfGammaCompleteValues () const; - double getScaleOfGammaValues () const; - int getTableSize () const; + virtual ~GammaValues() override = default; + static Ptr create(int DoF, int max_size_table=500); + virtual const std::vector &getCompleteGammaValues() const = 0; + virtual const std::vector &getIncompleteGammaValues() const = 0; + virtual const std::vector &getGammaValues() const = 0; + virtual double getScaleOfGammaCompleteValues () const = 0; + virtual double getScaleOfGammaValues () const = 0; + virtual int getTableSize () const = 0; }; ////////////////////////////////////////// QUALITY /////////////////////////////////////////// @@ -223,9 +247,7 @@ public: * @model: Mat current model, e.g., H matrix. */ virtual Score getScore (const Mat &model) const = 0; - virtual Score getScore (const std::vector &/*errors*/) const { - CV_Error(cv::Error::StsNotImplemented, "getScore(errors)"); - } + virtual Score getScore (const std::vector& errors) const = 0; // get @inliers of the @model. Assume threshold is given // @inliers must be preallocated to maximum points size. virtual int getInliers (const Mat &model, std::vector &inliers) const = 0; @@ -236,11 +258,30 @@ public: // set @inliers_mask: true if point i is inlier, false - otherwise. virtual int getInliers (const Mat &model, std::vector &inliers_mask) const = 0; virtual int getPointsSize() const = 0; - virtual Ptr clone () const = 0; + virtual double getThreshold () const = 0; + virtual Ptr getErrorFnc () const = 0; static int getInliers (const Ptr &error, const Mat &model, std::vector &inliers_mask, double threshold); static int getInliers (const Ptr &error, const Mat &model, std::vector &inliers, double threshold); + static int getInliers (const std::vector &errors, std::vector &inliers, + double threshold); + static int getInliers (const std::vector &errors, std::vector &inliers, + double threshold); + Score selectBest (const std::vector &models, int num_models, Mat &best) { + if (num_models == 0) return {}; + int best_idx = 0; + Score best_score = getScore(models[0]); + for (int i = 1; i < num_models; i++) { + const auto sc = getScore(models[i]); + if (sc.isBetter(best_score)) { + best_score = sc; + best_idx = i; + } + } + models[best_idx].copyTo(best); + return best_score; + } }; // RANSAC (binary) quality @@ -252,16 +293,16 @@ public: // M-estimator quality - truncated Squared error class MsacQuality : public Quality { public: - static Ptr create(int points_size_, double threshold_, const Ptr &error_); + static Ptr create(int points_size_, double threshold_, const Ptr &error_, double k_msac=2.25); }; // Marginlizing Sample Consensus quality, D. Barath et al. class MagsacQuality : public Quality { public: static Ptr create(double maximum_thr, int points_size_,const Ptr &error_, + const Ptr &gamma_generator, double tentative_inlier_threshold_, int DoF, double sigma_quantile, - double upper_incomplete_of_sigma_quantile, - double lower_incomplete_of_sigma_quantile, double C_); + double upper_incomplete_of_sigma_quantile); }; // Least Median of Squares Quality @@ -273,31 +314,40 @@ public: ////////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////// DEGENERACY ////////////////////////////////// class Degeneracy : public Algorithm { +private: + Mat homogr; public: virtual ~Degeneracy() override = default; /* * Check if sample causes degenerate configurations. * For example, test if points are collinear. */ - virtual bool isSampleGood (const std::vector &/*sample*/) const { + virtual bool isSampleGood (const std::vector& sample) const { + CV_UNUSED(sample); return true; } /* * Check if model satisfies constraints. * For example, test if epipolar geometry satisfies oriented constraint. */ - virtual bool isModelValid (const Mat &/*model*/, const std::vector &/*sample*/) const { + virtual bool isModelValid (const Mat& model, const std::vector& sample) const { + CV_UNUSED(model); + CV_UNUSED(sample); return true; } /* * Fix degenerate model. * Return true if model is degenerate, false - otherwise */ - virtual bool recoverIfDegenerate (const std::vector &/*sample*/,const Mat &/*best_model*/, - Mat &/*non_degenerate_model*/, Score &/*non_degenerate_model_score*/) { + virtual bool recoverIfDegenerate (const std::vector &sample,const Mat &best_model, const Score &score, + Mat &non_degenerate_model, Score &non_degenerate_model_score) { + CV_UNUSED(sample); + CV_UNUSED(best_model); + CV_UNUSED(score); + CV_UNUSED(non_degenerate_model); + CV_UNUSED(non_degenerate_model_score); return false; } - virtual Ptr clone(int /*state*/) const { return makePtr(); } }; class EpipolarGeometryDegeneracy : public Degeneracy { @@ -316,17 +366,31 @@ public: static Ptr create(const Mat &points_); }; +class FundamentalDegeneracyViaE : public EpipolarGeometryDegeneracy { +public: + static Ptr create (const Ptr &quality, const Mat &pts, + const Mat &calib_pts, const Matx33d &K1, const Matx33d &K2, bool is_f_objective); +}; + class FundamentalDegeneracy : public EpipolarGeometryDegeneracy { public: - // threshold for homography is squared so is around 2.236 pixels + virtual void setPrincipalPoint (double px_, double py_) = 0; + virtual void setPrincipalPoint (double px_, double py_, double px2_, double py2_) = 0; + virtual bool verifyFundamental (const Mat &F_best, const Score &F_score, const std::vector &inliers_mask, cv::Mat &F_new, Score &new_score) = 0; static Ptr create (int state, const Ptr &quality_, - const Mat &points_, int sample_size_, double homography_threshold); + const Mat &points_, int sample_size_, int max_iters_plane_and_parallax, + double homography_threshold, double f_inlier_thr_sqr, const Mat true_K1=Mat(), const Mat true_K2=Mat()); }; ///////////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////// ESTIMATOR ////////////////////////////////// class Estimator : public Algorithm{ public: + virtual int estimateModelNonMinimalSample (const Mat& model, const std::vector &sample, int sample_size, std::vector + &models, const std::vector &weights) const { + CV_UNUSED(model); + return estimateModelNonMinimalSample(sample, sample_size, models, weights); + } /* * Estimate models with minimal solver. * Return number of valid solutions after estimation. @@ -352,7 +416,7 @@ public: virtual int getMaxNumSolutions () const = 0; // return maximum number of possible solutions of non-minimal estimation. virtual int getMaxNumSolutionsNonMinimal () const = 0; - virtual Ptr clone() const = 0; + virtual void enforceRankConstraint (bool enforce) = 0; }; class HomographyEstimator : public Estimator { @@ -368,7 +432,7 @@ public: }; class EssentialEstimator : public Estimator { -public: +public : static Ptr create (const Ptr &min_solver_, const Ptr &non_min_solver_, const Ptr °eneracy_); }; @@ -391,15 +455,12 @@ class ModelVerifier : public Algorithm { public: virtual ~ModelVerifier() override = default; // Return true if model is good, false - otherwise. - virtual bool isModelGood(const Mat &model) = 0; - // Return true if score was computed during evaluation. - virtual bool getScore(Score &score) const = 0; + virtual bool isModelGood(const Mat &model, Score &score) = 0; // update verifier by given inlier number - virtual void update (int highest_inlier_number) = 0; - virtual const std::vector &getErrors() const = 0; - virtual bool hasErrors () const = 0; - virtual Ptr clone (int state) const = 0; - static Ptr create(); + virtual void update (const Score &score, int iteration) = 0; + virtual void reset() = 0; + virtual void updateSPRT (double time_model_est, double time_corr_ver, double new_avg_models, double new_delta, double new_epsilon, const Score &best_score) = 0; + static Ptr create(const Ptr &qualtiy); }; struct SPRT_history { @@ -421,7 +482,7 @@ struct SPRT_history { double epsilon, delta, A; // number of samples processed by test int tested_samples; // k - SPRT_history () : epsilon(0), delta(0), A(0) { + SPRT_history () { tested_samples = 0; } }; @@ -431,14 +492,14 @@ struct SPRT_history { * Matas, Jiri, and Ondrej Chum. "Randomized RANSAC with sequential probability ratio test." * Tenth IEEE International Conference on Computer Vision (ICCV'05) Volume 1. Vol. 2. IEEE, 2005. */ -class SPRT : public ModelVerifier { +class AdaptiveSPRT : public ModelVerifier { public: - // return constant reference of vector of SPRT histories for SPRT termination. virtual const std::vector &getSPRTvector () const = 0; - static Ptr create (int state, const Ptr &err_, int points_size_, - double inlier_threshold_, double prob_pt_of_good_model, - double prob_pt_of_bad_model, double time_sample, double avg_num_models, - ScoreMethod score_type_); + virtual int avgNumCheckedPts () const = 0; + static Ptr create (int state, const Ptr &quality, int points_size_, + double inlier_threshold_, double prob_pt_of_good_model, double prob_pt_of_bad_model, + double time_sample, double avg_num_models, ScoreMethod score_type_, + double k_mlesac, bool is_adaptive = true); }; ////////////////////////////////////////////////////////////////////////////////////////// @@ -450,7 +511,6 @@ public: virtual void setNewPointsSize (int points_size) = 0; // generate sample. Fill @sample with indices of points. virtual void generateSample (std::vector &sample) = 0; - virtual Ptr clone (int state) const = 0; }; //////////////////////////////////////////////////////////////////////////////////////////////// @@ -460,6 +520,7 @@ public: virtual ~NeighborhoodGraph() override = default; // Return neighbors of the point with index @point_idx_ in the graph. virtual const std::vector &getNeighbors(int point_idx_) const = 0; + virtual const std::vector> &getGraph () const = 0; }; class RadiusSearchNeighborhoodGraph : public NeighborhoodGraph { @@ -481,6 +542,11 @@ public: int cell_size_x_img1_, int cell_size_y_img1_, int cell_size_x_img2_, int cell_size_y_img2_, int max_neighbors); }; +class GridNeighborhoodGraph2Images : public NeighborhoodGraph { +public: + static Ptr create(const Mat &points, int points_size, + float cell_size_x_img1_, float cell_size_y_img1_, float cell_size_x_img2_, float cell_size_y_img2_); +}; ////////////////////////////////////// UNIFORM SAMPLER //////////////////////////////////////////// class UniformSampler : public Sampler { @@ -488,6 +554,11 @@ public: static Ptr create(int state, int sample_size_, int points_size_); }; +class QuasiUniformSampler : public Sampler { +public: + static Ptr create(int state, int sample_size_, int points_size_); +}; + /////////////////////////////////// PROSAC (SIMPLE) SAMPLER /////////////////////////////////////// class ProsacSimpleSampler : public Sampler { public: @@ -523,60 +594,74 @@ public: ///////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////// TERMINATION /////////////////////////////////////////// -class TerminationCriteria : public Algorithm { +class Termination : public Algorithm { public: // update termination object by given @model and @inlier number. // and return maximum number of predicted iteration - virtual int update(const Mat &model, int inlier_number) = 0; - // clone termination - virtual Ptr clone () const = 0; + virtual int update(const Mat &model, int inlier_number) const = 0; }; //////////////////////////////// STANDARD TERMINATION /////////////////////////////////////////// -class StandardTerminationCriteria : public TerminationCriteria { +class StandardTerminationCriteria : public Termination { public: static Ptr create(double confidence, int points_size_, int sample_size_, int max_iterations_); }; ///////////////////////////////////// SPRT TERMINATION ////////////////////////////////////////// -class SPRTTermination : public TerminationCriteria { +class SPRTTermination : public Termination { public: - static Ptr create(const std::vector &sprt_histories_, + static Ptr create(const Ptr &sprt, double confidence, int points_size_, int sample_size_, int max_iterations_); }; ///////////////////////////// PROGRESSIVE-NAPSAC-SPRT TERMINATION ///////////////////////////////// -class SPRTPNapsacTermination : public TerminationCriteria { +class SPRTPNapsacTermination : public Termination { public: - static Ptr create(const std::vector& - sprt_histories_, double confidence, int points_size_, int sample_size_, + static Ptr create(const Ptr & + sprt, double confidence, int points_size_, int sample_size_, int max_iterations_, double relax_coef_); }; ////////////////////////////////////// PROSAC TERMINATION ///////////////////////////////////////// -class ProsacTerminationCriteria : public TerminationCriteria { +class ProsacTerminationCriteria : public Termination { public: + virtual const std::vector &getNonRandomInliers () const = 0; + virtual int updateTerminationLength (const Mat &model, int inliers_size, int &found_termination_length) const = 0; static Ptr create(const Ptr &sampler_, const Ptr &error_, int points_size_, int sample_size, double confidence, int max_iters, int min_termination_length, double beta, double non_randomness_phi, - double inlier_thresh); + double inlier_thresh, const std::vector &non_rand_inliers); }; ////////////////////////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////// UTILS //////////////////////////////////////////////// namespace Utils { + void densitySort (const Mat &points, int knn, Mat &sorted_points, std::vector &sorted_mask); /* * calibrate points: [x'; 1] = K^-1 [x; 1] * @points is matrix N x 4. * @norm_points is output matrix N x 4 with calibrated points. */ - void calibratePoints (const Matx33d &K1, const Matx33d &K2, const Mat &points, Mat &norm_points); - void calibrateAndNormalizePointsPnP (const Matx33d &K, const Mat &pts, Mat &calib_norm_pts); - void normalizeAndDecalibPointsPnP (const Matx33d &K, Mat &pts, Mat &calib_norm_pts); - void decomposeProjection (const Mat &P, Matx33d &K_, Mat &R, Mat &t, bool same_focal=false); - double getCalibratedThreshold (double threshold, const Matx33d &K1, const Matx33d &K2); + void calibratePoints (const Mat &K1, const Mat &K2, const Mat &points, Mat &norm_points); + void calibrateAndNormalizePointsPnP (const Mat &K, const Mat &pts, Mat &calib_norm_pts); + void normalizeAndDecalibPointsPnP (const Mat &K, Mat &pts, Mat &calib_norm_pts); + void decomposeProjection (const Mat &P, Matx33d &K_, Matx33d &R, Vec3d &t, bool same_focal=false); + double getCalibratedThreshold (double threshold, const Mat &K1, const Mat &K2); float findMedian (std::vector &array); + double intersectionOverUnion (const std::vector &a, const std::vector &b); + void triangulatePoints (const Mat &points, const Mat &E_, const Mat &K1, const Mat &K2, Mat &points3D, Mat &R, Mat &t, + std::vector &good_mask, std::vector &depths1, std::vector &depths2); + void triangulatePoints (const Mat &E, const Mat &points1, const Mat &points2, Mat &corr_points1, Mat &corr_points2, + const Mat &K1, const Mat &K2, Mat &points3D, Mat &R, Mat &t, const std::vector &good_point_mask); + int triangulatePointsRt (const Mat &points, Mat &points3D, const Mat &K1_, const Mat &K2_, + const cv::Mat &R, const cv::Mat &t_vec, std::vector &good_mask, std::vector &depths1, std::vector &depths2); + int decomposeHomography (const Matx33d &Hnorm, std::vector &R, std::vector &t); + double getPoissonCDF (double lambda, int tentative_inliers); + void getClosePoints (const cv::Mat &points, std::vector> &close_points, float close_thr_sqr); + Vec3d getLeftEpipole (const Mat &F); + Vec3d getRightEpipole (const Mat &F); + int removeClosePoints (const Mat &points, Mat &new_points, float thr); } namespace Math { // return skew symmetric matrix @@ -587,6 +672,12 @@ namespace Math { Vec3d rotMat2RotVec (const Matx33d &R); } +class SolverPoly: public Algorithm { +public: + virtual int getRealRoots (const std::vector &coeffs, std::vector &real_roots) = 0; + static Ptr create(); +}; + ///////////////////////////////////////// RANDOM GENERATOR ///////////////////////////////////// class RandomGenerator : public Algorithm { public: @@ -609,7 +700,6 @@ public: virtual int getRandomNumber (int max_rng) = 0; virtual const std::vector &generateUniqueRandomSubset (std::vector &array1, int size1) = 0; - virtual Ptr clone (int state) const = 0; }; class UniformRandomGenerator : public RandomGenerator { @@ -618,6 +708,24 @@ public: static Ptr create (int state, int max_range, int subset_size_); }; +class WeightFunction : public Algorithm { +public: + virtual int getInliersWeights (const std::vector &errors, std::vector &inliers, std::vector &weights) const = 0; + virtual int getInliersWeights (const std::vector &errors, std::vector &inliers, std::vector &weights, double thr_sqr) const = 0; + virtual double getThreshold() const = 0; +}; + +class GaussWeightFunction : public WeightFunction { +public: + static Ptr create (double thr, double sigma, double outlier_prob); +}; + +class MagsacWeightFunction : public WeightFunction { +public: + static Ptr create (const Ptr &gamma_generator_, + int DoF_, double upper_incomplete_of_sigma_quantile, double C_, double max_sigma_); +}; + /////////////////////////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////// LOCAL OPTIMIZATION ///////////////////////////////////////// class LocalOptimization : public Algorithm { @@ -632,17 +740,18 @@ public: */ virtual bool refineModel (const Mat &best_model, const Score &best_model_score, Mat &new_model, Score &new_model_score) = 0; - virtual Ptr clone(int state) const = 0; + virtual void setCurrentRANSACiter (int /*ransac_iter*/) {} + virtual int getNumLOoptimizations () const { return 0; } }; //////////////////////////////////// GRAPH CUT LO //////////////////////////////////////// class GraphCut : public LocalOptimization { public: static Ptr - create(const Ptr &estimator_, const Ptr &error_, + create(const Ptr &estimator_, const Ptr &quality_, const Ptr &neighborhood_graph_, const Ptr &lo_sampler_, double threshold_, - double spatial_coherence_term, int gc_iters); + double spatial_coherence_term, int gc_iters, Ptr termination_= nullptr); }; //////////////////////////////////// INNER + ITERATIVE LO /////////////////////////////////////// @@ -655,14 +764,13 @@ public: int lo_iter_max_iterations, double threshold_multiplier); }; -class SigmaConsensus : public LocalOptimization { +class SimpleLocalOptimization : public LocalOptimization { public: - static Ptr - create(const Ptr &estimator_, const Ptr &error_, - const Ptr &quality, const Ptr &verifier_, - int max_lo_sample_size, int number_of_irwls_iters_, - int DoF, double sigma_quantile, double upper_incomplete_of_sigma_quantile, - double C_, double maximum_thr); + static Ptr create + (const Ptr &quality_, const Ptr &estimator_, + const Ptr termination_, const Ptr &random_gen, + const Ptr weight_fnc_, + int max_lo_iters_, double inlier_thr_sqr, bool updated_lo=false); }; /////////////////////////////////////////////////////////////////////////////////////////////////// @@ -681,35 +789,86 @@ public: Mat &new_model, Score &new_model_score) = 0; }; -///////////////////////////////////// LEAST SQUARES POLISHER ////////////////////////////////////// -class LeastSquaresPolishing : public FinalModelPolisher { +class NonMinimalPolisher : public FinalModelPolisher { public: - static Ptr create (const Ptr &estimator_, - const Ptr &quality_, int lsq_iterations); + static Ptr create(const Ptr &quality_, const Ptr &solver_, + Ptr weight_fnc_, int max_iters_, double iou_thr_); }; +class CovarianceSolver : public NonMinimalSolver { +public: + ~CovarianceSolver() override = default; + int estimate (const std::vector &, int , std::vector &, + const std::vector &) const override { + assert(false && "not implemeted!"); + return 0; + } + virtual int estimate (const std::vector &new_mask, std::vector &models, + const std::vector &weights) override = 0; + virtual void reset() = 0; +}; +class CovarianceEpipolarSolver : public CovarianceSolver { +public: + static Ptr create (const Mat &points, bool is_fundamental); + static Ptr create (const Mat &points, const Matx33d &T1, const Matx33d &T2); +}; +class CovarianceHomographySolver : public CovarianceSolver { +public: + static Ptr create (const Mat &points); + static Ptr create (const Mat &points, const Matx33d &T1, const Matx33d &T2); +}; +class CovarianceAffineSolver : public CovarianceSolver { +public: + static Ptr create (const Mat &points, const Matx33d &T1, const Matx33d &T2); + static Ptr create (const Mat &points); +}; + +/////////////////////////////////// POSE LIB //////////////////////////////////////// +struct CameraPose { + cv::Matx33d R; + cv::Vec3d t; + double alpha = 1.0; // either focal length or scale +}; +typedef std::vector CameraPoseVector; + +struct BundleOptions { + int max_iterations = 100; + enum LossType { + MLESAC, + } loss_type = LossType::MLESAC; // CAUCHY; + double loss_scale = 1.0; + double gradient_tol = 1e-8; + double step_tol = 1e-8; + double initial_lambda = 1e-3; +}; + +bool satisfyCheirality (const cv::Matx33d& R, const cv::Vec3d &t, const cv::Vec3d &x1, const cv::Vec3d &x2); + +// Relative pose refinement. Minimizes Sampson error error. Assumes identity intrinsics (calibrated camera) +// Returns number of iterations. +int refine_relpose(const cv::Mat &correspondences_, + const std::vector &sample_, + const int sample_size_, + CameraPose *pose, + const BundleOptions &opt = BundleOptions(), + const double *weights = nullptr); /////////////////////////////////// RANSAC OUTPUT /////////////////////////////////// class RansacOutput : public Algorithm { public: virtual ~RansacOutput() override = default; static Ptr create(const Mat &model_, - const std::vector &inliers_mask_, - int time_mcs_, double score_, int number_inliers_, int number_iterations_, - int number_estimated_models_, int number_good_models_); + const std::vector &inliers_mask_, int number_inliers_, + int number_iterations_, ModelConfidence conf, const std::vector &errors_); // Return inliers' indices. size of vector = number of inliers virtual const std::vector &getInliers() = 0; // Return inliers mask. Vector of points size. 1-inlier, 0-outlier. virtual const std::vector &getInliersMask() const = 0; - virtual int getTimeMicroSeconds() const = 0; - virtual int getTimeMicroSeconds1() const = 0; - virtual int getTimeMilliSeconds2() const = 0; - virtual int getTimeSeconds3() const = 0; virtual int getNumberOfInliers() const = 0; - virtual int getNumberOfMainIterations() const = 0; - virtual int getNumberOfGoodModels () const = 0; - virtual int getNumberOfEstimatedModels () const = 0; virtual const Mat &getModel() const = 0; + virtual int getNumberOfIters() const = 0; + virtual ModelConfidence getConfidence() const = 0; + virtual const std::vector &getResiduals() const = 0; }; ////////////////////////////////////////////// MODEL ///////////////////////////////////////////// @@ -724,7 +883,6 @@ public: // getters virtual int getSampleSize () const = 0; virtual bool isParallel() const = 0; - virtual int getMaxNumHypothesisToTestBeforeRejection() const = 0; virtual PolishingMethod getFinalPolisher () const = 0; virtual LocalOptimMethod getLO () const = 0; virtual ErrorMetric getError () const = 0; @@ -744,6 +902,7 @@ public: virtual int getCellSize () const = 0; virtual int getGraphRadius() const = 0; virtual double getRelaxCoef () const = 0; + virtual bool isNonRandomnessTest () const = 0; virtual int getFinalLSQIterations () const = 0; virtual int getDegreesOfFreedom () const = 0; @@ -753,6 +912,7 @@ public: virtual double getC () const = 0; virtual double getMaximumThreshold () const = 0; virtual double getGraphCutSpatialCoherenceTerm () const = 0; + virtual double getKmlesac () const = 0; virtual int getLOSampleSize () const = 0; virtual int getLOThresholdMultiplier() const = 0; virtual int getLOIterativeSampleSize() const = 0; @@ -760,9 +920,15 @@ public: virtual int getLOInnerMaxIters() const = 0; virtual const std::vector &getGridCellNumber () const = 0; virtual int getRandomGeneratorState () const = 0; - virtual int getMaxItersBeforeLO () const = 0; + virtual MethodSolver getRansacSolver () const = 0; + virtual int getPlaneAndParallaxIters () const = 0; + virtual int getLevMarqIters () const = 0; + virtual int getLevMarqItersLO () const = 0; + virtual bool isLarssonOptimization () const = 0; + virtual int getProsacMaxSamples() const = 0; // setters + virtual void setNonRandomnessTest (bool set) = 0; virtual void setLocalOptimization (LocalOptimMethod lo_) = 0; virtual void setKNearestNeighhbors (int knn_) = 0; virtual void setNeighborsType (NeighborSearchMethod neighbors) = 0; @@ -774,8 +940,8 @@ public: virtual void setLOIterations (int iters) = 0; virtual void setLOIterativeIters (int iters) = 0; virtual void setLOSampleSize (int lo_sample_size) = 0; - virtual void setThresholdMultiplierLO (double thr_mult) = 0; virtual void setRandomGeneratorState (int state) = 0; + virtual void setFinalLSQ (int iters) = 0; virtual void maskRequired (bool required) = 0; virtual bool isMaskRequired () const = 0; @@ -783,6 +949,9 @@ public: double confidence_=0.95, int max_iterations_=5000, ScoreMethod score_ =ScoreMethod::SCORE_METHOD_MSAC); }; +double getLambda (std::vector &supports, double cdf_thr, int points_size, + int sample_size, bool is_indendent_inliers, int &min_non_random_inliers); + Mat findHomography(InputArray srcPoints, InputArray dstPoints, int method, double ransacReprojThreshold, OutputArray mask, const int maxIters, const double confidence); @@ -811,7 +980,7 @@ Mat estimateAffine2D(InputArray from, InputArray to, OutputArray inliers, void saveMask (OutputArray mask, const std::vector &inliers_mask); void setParameters (Ptr ¶ms, EstimationMethod estimator, const UsacParams &usac_params, bool mask_need); -bool run (const Ptr ¶ms, InputArray points1, InputArray points2, int state, +bool run (const Ptr ¶ms, InputArray points1, InputArray points2, Ptr &ransac_output, InputArray K1_, InputArray K2_, InputArray dist_coeff1, InputArray dist_coeff2); }} diff --git a/modules/calib3d/src/usac/bundle.cpp b/modules/calib3d/src/usac/bundle.cpp new file mode 100644 index 0000000000..a621490575 --- /dev/null +++ b/modules/calib3d/src/usac/bundle.cpp @@ -0,0 +1,308 @@ +// Copyright (c) 2020, Viktor Larsson +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "../precomp.hpp" +#include "../usac.hpp" + +namespace cv { namespace usac { +class MlesacLoss { + public: + MlesacLoss(double threshold) : squared_thr(threshold * threshold), norm_thr(squared_thr*3), one_over_thr(1/norm_thr), inv_sq_thr(1/squared_thr) {} + double loss(double r2) const { + return r2 < norm_thr ? r2 * one_over_thr - 1 : 0; + } + double weight(double r2) const { + // use Cauchly weight + return 1.0 / (1.0 + r2 * inv_sq_thr); + } + const double squared_thr; + private: + const double norm_thr, one_over_thr, inv_sq_thr; +}; + +class RelativePoseJacobianAccumulator { +private: + const Mat* correspondences; + const std::vector &sample; + const int sample_size; + const MlesacLoss &loss_fn; + const double *weights; + +public: + RelativePoseJacobianAccumulator( + const Mat& correspondences_, + const std::vector &sample_, + const int sample_size_, + const MlesacLoss &l, + const double *w = nullptr) : + correspondences(&correspondences_), + sample(sample_), + sample_size(sample_size_), + loss_fn(l), + weights(w) {} + + Matx33d essential_from_motion(const CameraPose &pose) const { + return Matx33d(0.0, -pose.t(2), pose.t(1), + pose.t(2), 0.0, -pose.t(0), + -pose.t(1), pose.t(0), 0.0) * pose.R; + } + + double residual(const CameraPose &pose) const { + const Matx33d E = essential_from_motion(pose); + const float m11=static_cast(E(0,0)), m12=static_cast(E(0,1)), m13=static_cast(E(0,2)); + const float m21=static_cast(E(1,0)), m22=static_cast(E(1,1)), m23=static_cast(E(1,2)); + const float m31=static_cast(E(2,0)), m32=static_cast(E(2,1)), m33=static_cast(E(2,2)); + const auto * const pts = (float *) correspondences->data; + double cost = 0.0; + for (int k = 0; k < sample_size; ++k) { + const int idx = 4*sample[k]; + const float x1=pts[idx], y1=pts[idx+1], x2=pts[idx+2], y2=pts[idx+3]; + const float F_pt1_x = m11 * x1 + m12 * y1 + m13, + F_pt1_y = m21 * x1 + m22 * y1 + m23; + const float pt2_F_x = x2 * m11 + y2 * m21 + m31, + pt2_F_y = x2 * m12 + y2 * m22 + m32; + const float pt2_F_pt1 = x2 * F_pt1_x + y2 * F_pt1_y + m31 * x1 + m32 * y1 + m33; + const float r2 = pt2_F_pt1 * pt2_F_pt1 / (F_pt1_x * F_pt1_x + F_pt1_y * F_pt1_y + + pt2_F_x * pt2_F_x + pt2_F_y * pt2_F_y); + if (weights == nullptr) + cost += loss_fn.loss(r2); + else cost += weights[k] * loss_fn.loss(r2); + } + return cost; + } + + void accumulate(const CameraPose &pose, Matx &JtJ, Matx &Jtr, Matx &tangent_basis) const { + const auto * const pts = (float *) correspondences->data; + // We start by setting up a basis for the updates in the translation (orthogonal to t) + // We find the minimum element of t and cross product with the corresponding basis vector. + // (this ensures that the first cross product is not close to the zero vector) + Vec3d tangent_basis_col0; + if (std::abs(pose.t(0)) < std::abs(pose.t(1))) { + // x < y + if (std::abs(pose.t(0)) < std::abs(pose.t(2))) { + tangent_basis_col0 = pose.t.cross(Vec3d(1,0,0)); + } else { + tangent_basis_col0 = pose.t.cross(Vec3d(0,0,1)); + } + } else { + // x > y + if (std::abs(pose.t(1)) < std::abs(pose.t(2))) { + tangent_basis_col0 = pose.t.cross(Vec3d(0,1,0)); + } else { + tangent_basis_col0 = pose.t.cross(Vec3d(0,0,1)); + } + } + tangent_basis_col0 /= norm(tangent_basis_col0); + Vec3d tangent_basis_col1 = pose.t.cross(tangent_basis_col0); + tangent_basis_col1 /= norm(tangent_basis_col1); + for (int i = 0; i < 3; i++) { + tangent_basis(i,0) = tangent_basis_col0(i); + tangent_basis(i,1) = tangent_basis_col1(i); + } + + const Matx33d E = essential_from_motion(pose); + + // Matrices contain the jacobians of E w.r.t. the rotation and translation parameters + // Each column is vec(E*skew(e_k)) where e_k is k:th basis vector + const Matx dR = {0., -E(0,2), E(0,1), + 0., -E(1,2), E(1,1), + 0., -E(2,2), E(2,1), + E(0,2), 0., -E(0,0), + E(1,2), 0., -E(1,0), + E(2,2), 0., -E(2,0), + -E(0,1), E(0,0), 0., + -E(1,1), E(1,0), 0., + -E(2,1), E(2,0), 0.}; + + Matx dt; + // Each column is vec(skew(tangent_basis[k])*R) + for (int i = 0; i <= 2; i+=1) { + const Vec3d r_i(pose.R(0,i), pose.R(1,i), pose.R(2,i)); + for (int j = 0; j <= 1; j+= 1) { + const Vec3d v = (j == 0 ? tangent_basis_col0 : tangent_basis_col1).cross(r_i); + for (int k = 0; k < 3; k++) { + dt(3*i+k,j) = v[k]; + } + } + } + + for (int k = 0; k < sample_size; ++k) { + const auto point_idx = 4*sample[k]; + const Vec3d pt1 (pts[point_idx], pts[point_idx+1], 1), pt2 (pts[point_idx+2], pts[point_idx+3], 1); + const double C = pt2.dot(E * pt1); + + // J_C is the Jacobian of the epipolar constraint w.r.t. the image points + const Vec4d J_C ((E.col(0).t() * pt2)[0], (E.col(1).t() * pt2)[0], (E.row(0) * pt1)[0], (E.row(1) * pt1)[0]); + const double nJ_C = norm(J_C); + const double inv_nJ_C = 1.0 / nJ_C; + const double r = C * inv_nJ_C; + + if (r*r > loss_fn.squared_thr) continue; + + // Compute weight from robust loss function (used in the IRLS) + double weight = loss_fn.weight(r * r) / sample_size; + if (weights != nullptr) + weight = weights[k] * weight; + + if(weight < DBL_EPSILON) + continue; + + // Compute Jacobian of Sampson error w.r.t the fundamental/essential matrix (3x3) + Matx dF (pt1(0) * pt2(0), pt1(0) * pt2(1), pt1(0), pt1(1) * pt2(0), pt1(1) * pt2(1), pt1(1), pt2(0), pt2(1), 1.0); + const double s = C * inv_nJ_C * inv_nJ_C; + dF(0) -= s * (J_C(2) * pt1(0) + J_C(0) * pt2(0)); + dF(1) -= s * (J_C(3) * pt1(0) + J_C(0) * pt2(1)); + dF(2) -= s * (J_C(0)); + dF(3) -= s * (J_C(2) * pt1(1) + J_C(1) * pt2(0)); + dF(4) -= s * (J_C(3) * pt1(1) + J_C(1) * pt2(1)); + dF(5) -= s * (J_C(1)); + dF(6) -= s * (J_C(2)); + dF(7) -= s * (J_C(3)); + dF *= inv_nJ_C; + + // and then w.r.t. the pose parameters (rotation + tangent basis for translation) + const Matx13d dFdR = dF * dR; + const Matx12d dFdt = dF * dt; + const Matx J (dFdR(0), dFdR(1), dFdR(2), dFdt(0), dFdt(1)); + + // Accumulate into JtJ and Jtr + Jtr += weight * C * inv_nJ_C * J.t(); + JtJ(0, 0) += weight * (J(0) * J(0)); + JtJ(1, 0) += weight * (J(1) * J(0)); + JtJ(1, 1) += weight * (J(1) * J(1)); + JtJ(2, 0) += weight * (J(2) * J(0)); + JtJ(2, 1) += weight * (J(2) * J(1)); + JtJ(2, 2) += weight * (J(2) * J(2)); + JtJ(3, 0) += weight * (J(3) * J(0)); + JtJ(3, 1) += weight * (J(3) * J(1)); + JtJ(3, 2) += weight * (J(3) * J(2)); + JtJ(3, 3) += weight * (J(3) * J(3)); + JtJ(4, 0) += weight * (J(4) * J(0)); + JtJ(4, 1) += weight * (J(4) * J(1)); + JtJ(4, 2) += weight * (J(4) * J(2)); + JtJ(4, 3) += weight * (J(4) * J(3)); + JtJ(4, 4) += weight * (J(4) * J(4)); + } + } +}; + +bool satisfyCheirality (const Matx33d& R, const Vec3d &t, const Vec3d &x1, const Vec3d &x2) { + // This code assumes that x1 and x2 are unit vectors + const auto Rx1 = R * x1; + // lambda_2 * x2 = R * ( lambda_1 * x1 ) + t + // [1 a; a 1] * [lambda1; lambda2] = [b1; b2] + // [lambda1; lambda2] = [1 -a; -a 1] * [b1; b2] / (1 - a*a) + const double a = -Rx1.dot(x2), b1 = -Rx1.dot(t), b2 = x2.dot(t); + // Note that we drop the factor 1.0/(1-a*a) since it is always positive. + return (b1 - a * b2 > 0) && (-a * b1 + b2 > 0); +} + +int refine_relpose(const Mat &correspondences_, + const std::vector &sample_, + const int sample_size_, + CameraPose *pose, + const BundleOptions &opt, + const double* weights) { + MlesacLoss loss_fn(opt.loss_scale); + RelativePoseJacobianAccumulator accum(correspondences_, sample_, sample_size_, loss_fn, weights); + // return lm_5dof_impl(accum, pose, opt); + + Matx JtJ; + Matx Jtr; + Matx tangent_basis; + Matx33d sw = Matx33d::zeros(); + double lambda = opt.initial_lambda; + + // Compute initial cost + double cost = accum.residual(*pose); + bool recompute_jac = true; + int iter; + for (iter = 0; iter < opt.max_iterations; ++iter) { + // We only recompute jacobian and residual vector if last step was successful + if (recompute_jac) { + std::fill(JtJ.val, JtJ.val+25, 0); + std::fill(Jtr.val, Jtr.val +5, 0); + accum.accumulate(*pose, JtJ, Jtr, tangent_basis); + if (norm(Jtr) < opt.gradient_tol) + break; + } + + // Add dampening + JtJ(0, 0) += lambda; + JtJ(1, 1) += lambda; + JtJ(2, 2) += lambda; + JtJ(3, 3) += lambda; + JtJ(4, 4) += lambda; + + Matx sol; + Matx JtJ_symm = JtJ; + for (int i = 0; i < 5; i++) + for (int j = i+1; j < 5; j++) + JtJ_symm(i,j) = JtJ(j,i); + + const bool success = solve(-JtJ_symm, Jtr, sol); + if (!success || norm(sol) < opt.step_tol) + break; + + Vec3d w (sol(0,0), sol(1,0), sol(2,0)); + const double theta = norm(w); + w /= theta; + const double a = std::sin(theta); + const double b = std::cos(theta); + sw(0, 1) = -w(2); + sw(0, 2) = w(1); + sw(1, 2) = -w(0); + sw(1, 0) = w(2); + sw(2, 0) = -w(1); + sw(2, 1) = w(0); + + CameraPose pose_new; + pose_new.R = pose->R + pose->R * (a * sw + (1 - b) * sw * sw); + // In contrast to the 6dof case, we don't apply R here + // (since this can already be added into tangent_basis) + pose_new.t = pose->t + Vec3d(Mat(tangent_basis * Matx21d(sol(3,0), sol(4,0)))); + double cost_new = accum.residual(pose_new); + + if (cost_new < cost) { + *pose = pose_new; + lambda /= 10; + cost = cost_new; + recompute_jac = true; + } else { + JtJ(0, 0) -= lambda; + JtJ(1, 1) -= lambda; + JtJ(2, 2) -= lambda; + JtJ(3, 3) -= lambda; + JtJ(4, 4) -= lambda; + lambda *= 10; + recompute_jac = false; + } + } + return iter; +} +}} \ No newline at end of file diff --git a/modules/calib3d/src/usac/degeneracy.cpp b/modules/calib3d/src/usac/degeneracy.cpp index 7963ef3911..a6e028d5f3 100644 --- a/modules/calib3d/src/usac/degeneracy.cpp +++ b/modules/calib3d/src/usac/degeneracy.cpp @@ -13,7 +13,7 @@ private: const int min_sample_size; public: explicit EpipolarGeometryDegeneracyImpl (const Mat &points_, int sample_size_) : - points_mat(&points_), points ((float*) points_.data), min_sample_size (sample_size_) {} + points_mat(&points_), points ((float*) points_mat->data), min_sample_size (sample_size_) {} /* * Do oriented constraint to verify if epipolar geometry is in front or behind the camera. * Return: true if all points are in front of the camers w.r.t. tested epipolar geometry - satisfies constraint. @@ -23,39 +23,27 @@ public: * e × x ~+ x'^T F */ inline bool isModelValid(const Mat &F_, const std::vector &sample) const override { - // F is of rank 2, taking cross product of two rows we obtain null vector of F - Vec3d ec_mat = F_.row(0).cross(F_.row(2)); - auto * ec = ec_mat.val; // of size 3x1 - - // e is zero vector, recompute e - if (ec[0] <= 1.9984e-15 && ec[0] >= -1.9984e-15 && - ec[1] <= 1.9984e-15 && ec[1] >= -1.9984e-15 && - ec[2] <= 1.9984e-15 && ec[2] >= -1.9984e-15) { - ec_mat = F_.row(1).cross(F_.row(2)); - ec = ec_mat.val; - } + const Vec3d ep = Utils::getRightEpipole(F_); + const auto * const e = ep.val; // of size 3x1 const auto * const F = (double *) F_.data; // without loss of generality, let the first point in sample be in front of the camera. int pt = 4*sample[0]; - // s1 = F11 * x2 + F21 * y2 + F31 * 1 - // s2 = e'_2 * 1 - e'_3 * y1 + // check only two first elements of vectors (e × x) and (x'^T F) + // s1 = (x'^T F)[0] = x2 * F11 + y2 * F21 + 1 * F31 + // s2 = (e × x)[0] = e'_2 * 1 - e'_3 * y1 // sign1 = s1 * s2 - const double sign1 = (F[0]*points[pt+2]+F[3]*points[pt+3]+F[6])*(ec[1]-ec[2]*points[pt+1]); + const double sign1 = (F[0]*points[pt+2]+F[3]*points[pt+3]+F[6])*(e[1]-e[2]*points[pt+1]); for (int i = 1; i < min_sample_size; i++) { pt = 4 * sample[i]; // if signum of the first point and tested point differs // then two points are on different sides of the camera. - if (sign1*(F[0]*points[pt+2]+F[3]*points[pt+3]+F[6])*(ec[1]-ec[2]*points[pt+1])<0) - return false; + if (sign1*(F[0]*points[pt+2]+F[3]*points[pt+3]+F[6])*(e[1]-e[2]*points[pt+1])<0) + return false; } return true; } - - Ptr clone(int /*state*/) const override { - return makePtr(*points_mat, min_sample_size); - } }; void EpipolarGeometryDegeneracy::recoverRank (Mat &model, bool is_fundamental_mat) { /* @@ -81,9 +69,10 @@ class HomographyDegeneracyImpl : public HomographyDegeneracy { private: const Mat * points_mat; const float * const points; + const float TOLERANCE = 2 * FLT_EPSILON; // 2 from area of triangle public: explicit HomographyDegeneracyImpl (const Mat &points_) : - points_mat(&points_), points ((float *)points_.data) {} + points_mat(&points_), points ((float *)points_mat->data) {} inline bool isSampleGood (const std::vector &sample) const override { const int smpl1 = 4*sample[0], smpl2 = 4*sample[1], smpl3 = 4*sample[2], smpl4 = 4*sample[3]; @@ -119,222 +108,580 @@ public: // Checks if points are not collinear // If area of triangle constructed with 3 points is less then threshold then points are collinear: // |x1 y1 1| |x1 y1 1| - // (1/2) det |x2 y2 1| = (1/2) det |x2-x1 y2-y1 0| = (1/2) det |x2-x1 y2-y1| < threshold - // |x3 y3 1| |x3-x1 y3-y1 0| |x3-x1 y3-y1| + // (1/2) det |x2 y2 1| = (1/2) det |x2-x1 y2-y1 0| = det |x2-x1 y2-y1| < 2 * threshold + // |x3 y3 1| |x3-x1 y3-y1 0| |x3-x1 y3-y1| // for points on the first image - if (fabsf((x2-x1) * (y3-y1) - (y2-y1) * (x3-x1)) * 0.5 < FLT_EPSILON) return false; //1,2,3 - if (fabsf((x2-x1) * (y4-y1) - (y2-y1) * (x4-x1)) * 0.5 < FLT_EPSILON) return false; //1,2,4 - if (fabsf((x3-x1) * (y4-y1) - (y3-y1) * (x4-x1)) * 0.5 < FLT_EPSILON) return false; //1,3,4 - if (fabsf((x3-x2) * (y4-y2) - (y3-y2) * (x4-x2)) * 0.5 < FLT_EPSILON) return false; //2,3,4 + if (fabsf((x2-x1) * (y3-y1) - (y2-y1) * (x3-x1)) < TOLERANCE) return false; //1,2,3 + if (fabsf((x2-x1) * (y4-y1) - (y2-y1) * (x4-x1)) < TOLERANCE) return false; //1,2,4 + if (fabsf((x3-x1) * (y4-y1) - (y3-y1) * (x4-x1)) < TOLERANCE) return false; //1,3,4 + if (fabsf((x3-x2) * (y4-y2) - (y3-y2) * (x4-x2)) < TOLERANCE) return false; //2,3,4 // for points on the second image - if (fabsf((X2-X1) * (Y3-Y1) - (Y2-Y1) * (X3-X1)) * 0.5 < FLT_EPSILON) return false; //1,2,3 - if (fabsf((X2-X1) * (Y4-Y1) - (Y2-Y1) * (X4-X1)) * 0.5 < FLT_EPSILON) return false; //1,2,4 - if (fabsf((X3-X1) * (Y4-Y1) - (Y3-Y1) * (X4-X1)) * 0.5 < FLT_EPSILON) return false; //1,3,4 - if (fabsf((X3-X2) * (Y4-Y2) - (Y3-Y2) * (X4-X2)) * 0.5 < FLT_EPSILON) return false; //2,3,4 + if (fabsf((X2-X1) * (Y3-Y1) - (Y2-Y1) * (X3-X1)) < TOLERANCE) return false; //1,2,3 + if (fabsf((X2-X1) * (Y4-Y1) - (Y2-Y1) * (X4-X1)) < TOLERANCE) return false; //1,2,4 + if (fabsf((X3-X1) * (Y4-Y1) - (Y3-Y1) * (X4-X1)) < TOLERANCE) return false; //1,3,4 + if (fabsf((X3-X2) * (Y4-Y2) - (Y3-Y2) * (X4-X2)) < TOLERANCE) return false; //2,3,4 return true; } - Ptr clone(int /*state*/) const override { - return makePtr(*points_mat); - } }; Ptr HomographyDegeneracy::create (const Mat &points_) { return makePtr(points_); } +class FundamentalDegeneracyViaEImpl : public FundamentalDegeneracyViaE { +private: + bool is_F_objective; + std::vector> instances = {{0,1,2,3,4}, {2,3,4,5,6}, {0,1,4,5,6}}; + std::vector e_sample; + const Ptr quality; + Ptr e_degen, f_degen; + Ptr e_solver; + std::vector e_models; + const int E_SAMPLE_SIZE = 5; + Matx33d K2_inv_t, K1_inv; +public: + FundamentalDegeneracyViaEImpl (const Ptr &quality_, const Mat &pts, const Mat &calib_pts, const Matx33d &K1, const Matx33d &K2, bool is_f_objective) + : quality(quality_) { + is_F_objective = is_f_objective; + e_solver = EssentialMinimalSolver5pts::create(calib_pts, false/*svd*/, true/*nister*/); + f_degen = is_F_objective ? EpipolarGeometryDegeneracy::create(pts, 7) : EpipolarGeometryDegeneracy::create(calib_pts, 7); + e_degen = EpipolarGeometryDegeneracy::create(calib_pts, E_SAMPLE_SIZE); + e_sample = std::vector(E_SAMPLE_SIZE); + if (is_f_objective) { + K2_inv_t = K2.inv().t(); + K1_inv = K1.inv(); + } + } + bool isModelValid (const Mat &F, const std::vector &sample) const override { + return f_degen->isModelValid(F, sample); + } + bool recoverIfDegenerate (const std::vector &sample_7pt, const Mat &/*best*/, const Score &best_score, + Mat &out_model, Score &out_score) override { + out_score = Score(); + for (const auto &instance : instances) { + for (int i = 0; i < E_SAMPLE_SIZE; i++) + e_sample[i] = sample_7pt[instance[i]]; + const int num_models = e_solver->estimate(e_sample, e_models); + for (int i = 0; i < num_models; i++) { + if (e_degen->isModelValid(e_models[i], e_sample)) { + const Mat model = is_F_objective ? Mat(K2_inv_t * Matx33d(e_models[i]) * K1_inv) : e_models[i]; + const auto sc = quality->getScore(model); + if (sc.isBetter(out_score)) { + out_score = sc; + model.copyTo(out_model); + } + } + } + if (out_score.isBetter(best_score)) break; + } + return true; + } +}; +Ptr FundamentalDegeneracyViaE::create (const Ptr &quality, const Mat &pts, + const Mat &calib_pts, const Matx33d &K1, const Matx33d &K2, bool is_f_objective) { + return makePtr(quality, pts, calib_pts, K1, K2, is_f_objective); +} ///////////////////////////////// Fundamental Matrix Degeneracy /////////////////////////////////// class FundamentalDegeneracyImpl : public FundamentalDegeneracy { private: RNG rng; const Ptr quality; + const Ptr f_error; + Ptr h_repr_quality; const float * const points; const Mat * points_mat; const Ptr h_reproj_error; + Ptr f_non_min_solver; Ptr h_non_min_solver; + Ptr random_gen_H; const EpipolarGeometryDegeneracyImpl ep_deg; // threshold to find inliers for homography model - const double homography_threshold, log_conf = log(0.05); + const double homography_threshold, log_conf = log(0.05), H_SAMPLE_THR_SQR = 49/*7^2*/, MAX_H_THR = 225/*15^2*/; + double f_threshold_sqr, best_focal = -1; // points (1-7) to verify in sample std::vector> h_sample {{0,1,2},{3,4,5},{0,1,6},{3,4,6},{2,5,6}}; - std::vector h_inliers; + std::vector> h_sample_ver {{3,4,5,6},{0,1,2,6},{2,3,4,5},{0,1,2,5},{0,1,3,4}}; + std::vector non_planar_supports, h_inliers, h_outliers, h_outliers_eval, f_inliers; std::vector weights; std::vector h_models; - const int points_size, sample_size; + const int points_size, max_iters_plane_and_parallax, MAX_H_SUBSET = 50, MAX_ITERS_H = 6; + int num_h_outliers, num_models_used_so_far = 0, estimated_min_non_planar_support, + num_h_outliers_eval, TENT_MIN_NON_PLANAR_SUPP; + const int MAX_MODELS_TO_TEST = 21, H_INLS_DEGEN_SAMPLE = 5; // 5 by DEGENSAC, Chum et al. + Matx33d K, K2, K_inv, K2_inv, K2_inv_t, true_K2_inv, true_K2_inv_t, true_K1_inv, true_K1, true_K2_t; + Score best_focal_score; + bool true_K_given, is_principal_pt_set = false; public: - FundamentalDegeneracyImpl (int state, const Ptr &quality_, const Mat &points_, - int sample_size_, double homography_threshold_) : - rng (state), quality(quality_), points((float *) points_.data), points_mat(&points_), + int sample_size_, int plane_and_parallax_iters, double homography_threshold_, + double f_inlier_thr_sqr, const Mat true_K1_, const Mat true_K2_) : + rng (state), quality(quality_), f_error(quality_->getErrorFnc()), points((float *) points_.data), points_mat(&points_), h_reproj_error(ReprojectionErrorForward::create(points_)), ep_deg (points_, sample_size_), homography_threshold (homography_threshold_), - points_size (quality_->getPointsSize()), sample_size (sample_size_) { + points_size (quality_->getPointsSize()), + max_iters_plane_and_parallax(plane_and_parallax_iters) { if (sample_size_ == 8) { // add more homography samples to test for 8-points F - h_sample.emplace_back(std::vector{0, 1, 7}); - h_sample.emplace_back(std::vector{0, 2, 7}); - h_sample.emplace_back(std::vector{3, 5, 7}); - h_sample.emplace_back(std::vector{3, 6, 7}); - h_sample.emplace_back(std::vector{2, 4, 7}); + h_sample.emplace_back(std::vector{0, 1, 7}); h_sample_ver.emplace_back(std::vector{2,3,4,5,6}); + h_sample.emplace_back(std::vector{0, 2, 7}); h_sample_ver.emplace_back(std::vector{1,3,4,5,6}); + h_sample.emplace_back(std::vector{3, 5, 7}); h_sample_ver.emplace_back(std::vector{0,1,2,4,6}); + h_sample.emplace_back(std::vector{3, 6, 7}); h_sample_ver.emplace_back(std::vector{0,1,2,4,5}); + h_sample.emplace_back(std::vector{2, 4, 7}); h_sample_ver.emplace_back(std::vector{0,1,3,5,6}); } + non_planar_supports = std::vector(MAX_MODELS_TO_TEST); h_inliers = std::vector(points_size); + h_outliers = std::vector(points_size); + h_outliers_eval = std::vector(points_size); + f_inliers = std::vector(points_size); h_non_min_solver = HomographyNonMinimalSolver::create(points_); + f_non_min_solver = EpipolarNonMinimalSolver::create(points_, true /*F*/); + num_h_outliers_eval = num_h_outliers = points_size; + f_threshold_sqr = f_inlier_thr_sqr; + h_repr_quality = MsacQuality::create(points_.rows, homography_threshold_, h_reproj_error); + true_K_given = ! true_K1_.empty() && ! true_K2_.empty(); + if (true_K_given) { + true_K1 = Matx33d((double *)true_K1_.data); + true_K2_inv = Matx33d(Mat(true_K2_.inv())); + true_K2_t = Matx33d(true_K2_).t(); + true_K1_inv = true_K1.inv(); + true_K2_inv_t = true_K2_inv.t(); + } + random_gen_H = UniformRandomGenerator::create(rng.uniform(0, INT_MAX), points_size, MAX_H_SUBSET); + estimated_min_non_planar_support = TENT_MIN_NON_PLANAR_SUPP = std::min(5, (int)(0.05*points_size)); + } + bool estimateHfrom3Points (const Mat &F_best, const std::vector &sample, Mat &H_best) { + Score H_best_score; + // find e', null vector of F^T + const Vec3d e_prime = Utils::getLeftEpipole(F_best); + const Matx33d A = Math::getSkewSymmetric(e_prime) * Matx33d(F_best); + bool is_degenerate = false; + int idx = -1; + for (const auto &h_i : h_sample) { // only 5 samples + idx++; + Matx33d H; + if (!getH(A, e_prime, 4 * sample[h_i[0]], 4 * sample[h_i[1]], 4 * sample[h_i[2]], H)) + continue; + h_reproj_error->setModelParameters(Mat(H)); + const auto &ver_pts = h_sample_ver[idx]; + int inliers_in_plane = 3; // 3 points are always inliers + for (int s : ver_pts) + if (h_reproj_error->getError(sample[s]) < homography_threshold) { + if (++inliers_in_plane >= H_INLS_DEGEN_SAMPLE) + break; + } + if (inliers_in_plane >= H_INLS_DEGEN_SAMPLE) { + is_degenerate = true; + const auto h_score = h_repr_quality->getScore(Mat(H)); + if (h_score.isBetter(H_best_score)) { + H_best_score = h_score; + H_best = Mat(H); + } + } + } + if (!is_degenerate) + return false; + int h_inls_cnt = optimizeH(H_best, H_best_score); + for (int iter = 0; iter < 2; iter++) { + if (h_non_min_solver->estimate(h_inliers, h_inls_cnt, h_models, weights) == 0) + break; + const auto h_score = h_repr_quality->getScore(h_models[0]); + if (h_score.isBetter(H_best_score)) { + H_best_score = h_score; + h_models[0].copyTo(H_best); + h_inls_cnt = h_repr_quality->getInliers(H_best, h_inliers); + } else break; + } + getOutliersH(H_best); + return true; + } + bool optimizeF (const Mat &F, const Score &score, Mat &F_new, Score &new_score) { + std::vector Fs; + if (f_non_min_solver->estimate(f_inliers, quality->getInliers(F, f_inliers), Fs, weights)) { + const auto F_polished_score = quality->getScore(f_error->getErrors(Fs[0])); + if (F_polished_score.isBetter(score)) { + Fs[0].copyTo(F_new); new_score = F_polished_score; + return true; + } + } + return false; + } + int optimizeH (Mat &H_best, Score &H_best_score) { + int h_inls_cnt = h_repr_quality->getInliers(H_best, h_inliers); + random_gen_H->setSubsetSize(h_inls_cnt <= MAX_H_SUBSET ? (int)(0.8*h_inls_cnt) : MAX_H_SUBSET); + if (random_gen_H->getSubsetSize() >= 4/*min H sample size*/) { + for (int iter = 0; iter < MAX_ITERS_H; iter++) { + if (h_non_min_solver->estimate(random_gen_H->generateUniqueRandomSubset(h_inliers, h_inls_cnt), random_gen_H->getSubsetSize(), h_models, weights) == 0) + continue; + const auto h_score = h_repr_quality->getScore(h_models[0]); + if (h_score.isBetter(H_best_score)) { + h_models[0].copyTo(H_best); + // if more inliers than previous best + if (h_score.inlier_number > H_best_score.inlier_number || h_score.inlier_number >= MAX_H_SUBSET) { + h_inls_cnt = h_repr_quality->getInliers(H_best, h_inliers); + random_gen_H->setSubsetSize(h_inls_cnt <= MAX_H_SUBSET ? (int)(0.8*h_inls_cnt) : MAX_H_SUBSET); + } + H_best_score = h_score; + } + } + } + return h_inls_cnt; + } + + bool recoverIfDegenerate (const std::vector &sample, const Mat &F_best, const Score &F_best_score, + Mat &non_degenerate_model, Score &non_degenerate_model_score) override { + const auto swapF = [&] (const Mat &_F, const Score &_score) { + const auto non_min_solver = EpipolarNonMinimalSolver::create(*points_mat, true); + if (! optimizeF(_F, _score, non_degenerate_model, non_degenerate_model_score)) { + _F.copyTo(non_degenerate_model); + non_degenerate_model_score = _score; + } + }; + Mat F_from_H, F_from_E, H_best; Score F_from_H_score, F_from_E_score; + if (! estimateHfrom3Points(F_best, sample, H_best)) { + return false; // non degenerate + } + if (true_K_given) { + if (getFfromTrueK(H_best, F_from_H, F_from_H_score)) { + if (F_from_H_score.isBetter(F_from_E_score)) + swapF(F_from_H, F_from_H_score); + else swapF(F_from_E, F_from_E_score); + return true; + } else { + non_degenerate_model_score = Score(); + return true; // no translation + } + } + const int non_planar_support_degen_F = getNonPlanarSupport(F_best); + Score F_pl_par_score, F_calib_score; Mat F_pl_par, F_calib; + if (calibDegensac(H_best, F_calib, F_calib_score, non_planar_support_degen_F, F_best_score)) { + if (planeAndParallaxRANSAC(H_best, h_outliers, num_h_outliers, max_iters_plane_and_parallax, true, + F_best_score, non_planar_support_degen_F, F_pl_par, F_pl_par_score) && F_pl_par_score.isBetter(F_calib_score) + && getNonPlanarSupport(F_pl_par) > getNonPlanarSupport(F_calib)) { + swapF(F_pl_par, F_pl_par_score); + return true; + } + swapF(F_calib, F_calib_score); + return true; + } else { + if (planeAndParallaxRANSAC(H_best, h_outliers, num_h_outliers, max_iters_plane_and_parallax, true, + F_best_score, non_planar_support_degen_F, F_pl_par, F_pl_par_score)) { + swapF(F_pl_par, F_pl_par_score); + return true; + } + } + if (! isFDegenerate(non_planar_support_degen_F)) { + return false; + } + non_degenerate_model_score = Score(); + return true; + } + + // RANSAC with plane-and-parallax to find new Fundamental matrix + bool getFfromTrueK (const Matx33d &H, Mat &F_from_K, Score &F_from_K_score) { + std::vector R; std::vector t; + if (decomposeHomographyMat(true_K2_inv * H * true_K1, Matx33d::eye(), R, t, noArray()) == 1) { + // std::cout << "Warning: translation is zero!\n"; + return false; // is degenerate + } + // sign of translation does not make difference + const Mat F1 = Mat(true_K2_inv_t * Math::getSkewSymmetric(t[0]) * R[0] * true_K1_inv); + const Mat F2 = Mat(true_K2_inv_t * Math::getSkewSymmetric(t[1]) * R[1] * true_K1_inv); + const auto score_f1 = quality->getScore(f_error->getErrors(F1)), score_f2 = quality->getScore(f_error->getErrors(F2)); + if (score_f1.isBetter(score_f2)) { + F_from_K = F1; F_from_K_score = score_f1; + } else { + F_from_K = F2; F_from_K_score = score_f2; + } + return true; + } + bool planeAndParallaxRANSAC (const Matx33d &H, std::vector &non_planar_pts, int num_non_planar_pts, + int max_iters_pl_par, bool use_preemptive, const Score &score_degen_F, int non_planar_support_degen_F, + Mat &F_new, Score &F_new_score) { + if (num_non_planar_pts < 2) + return false; + num_models_used_so_far = 0; // reset estimation of lambda for plane-and-parallax + int max_iters = max_iters_pl_par, non_planar_support = 0, iters = 0; + std::vector F_good; + const double CLOSE_THR = 1.0; + for (; iters < max_iters; iters++) { + // draw two random points + int h_outlier1 = 4 * non_planar_pts[rng.uniform(0, num_non_planar_pts)]; + int h_outlier2 = 4 * non_planar_pts[rng.uniform(0, num_non_planar_pts)]; + while (h_outlier1 == h_outlier2) + h_outlier2 = 4 * non_planar_pts[rng.uniform(0, num_non_planar_pts)]; + + const auto x1 = points[h_outlier1], y1 = points[h_outlier1+1], X1 = points[h_outlier1+2], Y1 = points[h_outlier1+3]; + const auto x2 = points[h_outlier2], y2 = points[h_outlier2+1], X2 = points[h_outlier2+2], Y2 = points[h_outlier2+3]; + if ((fabsf(X1 - X2) < CLOSE_THR && fabsf(Y1 - Y2) < CLOSE_THR) || + (fabsf(x1 - x2) < CLOSE_THR && fabsf(y1 - y2) < CLOSE_THR)) + continue; + + // do plane and parallax with outliers of H + // F = [(p1' x Hp1) x (p2' x Hp2)]_x H = [e']_x H + Vec3d ep = (Vec3d(X1, Y1, 1).cross(H * Vec3d(x1, y1, 1))) // l1 = p1' x Hp1 + .cross((Vec3d(X2, Y2, 1).cross(H * Vec3d(x2, y2, 1))));// l2 = p2' x Hp2 + const Matx33d F = Math::getSkewSymmetric(ep) * H; + const auto * const f = F.val; + // check orientation constraint of epipolar lines + const bool valid = (f[0]*x1+f[1]*y1+f[2])*(ep[1]-ep[2]*Y1) * (f[0]*x2+f[1]*y2+f[2])*(ep[1]-ep[2]*Y2) > 0; + if (!valid) continue; + + const int num_f_inliers_of_h_outliers = getNonPlanarSupport(Mat(F), num_models_used_so_far >= MAX_MODELS_TO_TEST, non_planar_support); + if (non_planar_support < num_f_inliers_of_h_outliers) { + non_planar_support = num_f_inliers_of_h_outliers; + const double predicted_iters = log_conf / log(1 - pow(static_cast + (getNonPlanarSupport(Mat(F), non_planar_pts, num_non_planar_pts)) / num_non_planar_pts, 2)); + if (use_preemptive && ! std::isinf(predicted_iters) && predicted_iters < max_iters) + max_iters = static_cast(predicted_iters); + F_good = { F }; + } else if (non_planar_support == num_f_inliers_of_h_outliers) { + F_good.emplace_back(F); + } + } + + F_new_score = Score(); + for (const auto &F_ : F_good) { + const auto sc = quality->getScore(f_error->getErrors(Mat(F_))); + if (sc.isBetter(F_new_score)) { + F_new_score = sc; + F_new = Mat(F_); + } + } + if (F_new_score.isBetter(score_degen_F) && non_planar_support > non_planar_support_degen_F) + return true; + if (isFDegenerate(non_planar_support)) + return false; + return true; + } + bool calibDegensac (const Matx33d &H, Mat &F_new, Score &F_new_score, int non_planar_support_degen_F, const Score &F_degen_score) { + if (! is_principal_pt_set) { + // estimate principal points from coordinates + float px1 = 0, py1 = 0, px2 = 0, py2 = 0; + for (int i = 0; i < points_size; i++) { + const int idx = 4*i; + if (px1 < points[idx ]) px1 = points[idx ]; + if (py1 < points[idx+1]) py1 = points[idx+1]; + if (px2 < points[idx+2]) px2 = points[idx+2]; + if (py2 < points[idx+3]) py2 = points[idx+3]; + } + setPrincipalPoint((int)(px1/2)+1, (int)(py1/2)+1, (int)(px2/2)+1, (int)(py2/2)+1); + } + std::vector R; std::vector t; std::vector F_good; + std::vector best_f; + int non_planar_support_out = 0; + for (double f = 300; f <= 3000; f += 150.) { + K(0,0) = K(1,1) = K2(0,0) = K2(1,1) = f; + const double one_over_f = 1/f; + K_inv(0,0) = K_inv(1,1) = K2_inv(0,0) = K2_inv(1,1) = K2_inv_t(0,0) = K2_inv_t(1,1) = one_over_f; + K_inv(0,2) = -K(0,2)*one_over_f; K_inv(1,2) = -K(1,2)*one_over_f; + K2_inv_t(2,0) = K2_inv(0,2) = -K2(0,2)*one_over_f; K2_inv_t(2,1) = K2_inv(1,2) = -K2(1,2)*one_over_f; + if (decomposeHomographyMat(K2_inv * H * K, Matx33d::eye(), R, t, noArray()) == 1) continue; + Mat F1 = Mat(K2_inv_t * Math::getSkewSymmetric(Vec3d(t[0])) * Matx33d(R[0]) * K_inv); + Mat F2 = Mat(K2_inv_t * Math::getSkewSymmetric(Vec3d(t[2])) * Matx33d(R[2]) * K_inv); + int non_planar_f1 = getNonPlanarSupport(F1, true, non_planar_support_out), + non_planar_f2 = getNonPlanarSupport(F2, true, non_planar_support_out); + if (non_planar_f1 < non_planar_f2) { + non_planar_f1 = non_planar_f2; F1 = F2; + } + if (non_planar_support_out < non_planar_f1) { + non_planar_support_out = non_planar_f1; + F_good = {F1}; + best_f = { f }; + } else if (non_planar_support_out == non_planar_f1) { + F_good.emplace_back(F1); + best_f.emplace_back(f); + } + } + F_new_score = Score(); + for (int i = 0; i < (int) F_good.size(); i++) { + const auto sc = quality->getScore(f_error->getErrors(F_good[i])); + if (sc.isBetter(F_new_score)) { + F_new_score = sc; + F_good[i].copyTo(F_new); + if (sc.isBetter(best_focal_score)) { + best_focal = best_f[i]; // save best focal length + best_focal_score = sc; + } + } + } + if (F_degen_score.isBetter(F_new_score) && non_planar_support_out <= non_planar_support_degen_F) + return false; + + /* + // logarithmic search -> faster but less accurate + double f_min = 300, f_max = 3500; + while (f_max - f_min > 100) { + const double f_half = (f_max + f_min) * 0.5f, left_half = (f_min + f_half) * 0.5f, right_half = (f_half + f_max) * 0.5f; + const double inl_in_left = eval_f(left_half), inl_in_right = eval_f(right_half); + if (inl_in_left > inl_in_right) + f_max = f_half; + else f_min = f_half; + } + */ + return true; + } + void getOutliersH (const Mat &H_best) { + // get H outliers + num_h_outliers_eval = num_h_outliers = 0; + const auto &h_errors = h_reproj_error->getErrors(H_best); + for (int pt = 0; pt < points_size; pt++) + if (h_errors[pt] > H_SAMPLE_THR_SQR) { + h_outliers[num_h_outliers++] = pt; + if (h_errors[pt] > MAX_H_THR) + h_outliers_eval[num_h_outliers_eval++] = pt; + } + } + + bool verifyFundamental (const Mat &F_best, const Score &F_score, const std::vector &inliers_mask, Mat &F_new, Score &new_score) override { + const int f_sample_size = 3, max_H_iters = 5; // 3.52 = log(0.01) / log(1 - std::pow(0.9, 3)); + int num_f_inliers = 0; + std::vector inliers(points_size), f_sample(f_sample_size); + for (int i = 0; i < points_size; i++) if (inliers_mask[i]) inliers[num_f_inliers++] = i; + const auto sampler = UniformSampler::create(0, f_sample_size, num_f_inliers); + // find e', null space of F^T + const Vec3d e_prime = Utils::getLeftEpipole(F_best); + const Matx33d A = Math::getSkewSymmetric(e_prime) * Matx33d(F_best); + Score H_best_score; Mat H_best; + for (int iter = 0; iter < max_H_iters; iter++) { + sampler->generateSample(f_sample); + Matx33d H; + if (!getH(A, e_prime, 4*inliers[f_sample[0]], 4*inliers[f_sample[1]], 4*inliers[f_sample[2]], H)) + continue; + const auto h_score = h_repr_quality->getScore(Mat(H)); + if (h_score.isBetter(H_best_score)) { + H_best_score = h_score; H_best = Mat(H); + } + } + if (H_best.empty()) return false; // non-degenerate + optimizeH(H_best, H_best_score); + getOutliersH(H_best); + const int non_planar_support_best_F = getNonPlanarSupport(F_best); + const bool is_F_degen = isFDegenerate(non_planar_support_best_F); + Mat F_from_K; Score F_from_K_score; + bool success = false; + // generate non-degenerate F even though so-far-the-best one may not be degenerate + if (true_K_given) { + // use GT calibration + if (getFfromTrueK(H_best, F_from_K, F_from_K_score)) { + new_score = F_from_K_score; + F_from_K.copyTo(F_new); + success = true; + } + } else { + // use calibrated DEGENSAC + if (calibDegensac(H_best, F_from_K, F_from_K_score, non_planar_support_best_F, F_score)) { + new_score = F_from_K_score; + F_from_K.copyTo(F_new); + success = true; + } + } + if (!is_F_degen) { + return false; + } else if (success) // F is degenerate + return true; // but successfully recovered + + // recover degenerate F using plane-and-parallax + Score plane_parallax_score; Mat F_plane_parallax; + if (planeAndParallaxRANSAC(H_best, h_outliers, num_h_outliers, 20, true, + F_score, non_planar_support_best_F, F_plane_parallax, plane_parallax_score)) { + new_score = plane_parallax_score; + F_plane_parallax.copyTo(F_new); + return true; + } + // plane-and-parallax failed. A previous non-degenerate so-far-the-best model will be used instead + new_score = Score(); + return true; + } + void setPrincipalPoint (double px_, double py_) override { + setPrincipalPoint(px_, py_, 0, 0); + } + void setPrincipalPoint (double px_, double py_, double px2_, double py2_) override { + if (px_ > DBL_EPSILON && py_ > DBL_EPSILON) { + is_principal_pt_set = true; + K = {1, 0, px_, 0, 1, py_, 0, 0, 1}; + if (px2_ > DBL_EPSILON && py2_ > DBL_EPSILON) K2 = {1, 0, px2_, 0, 1, py2_, 0, 0, 1}; + else K2 = K; + K_inv = K2_inv = K2_inv_t = Matx33d::eye(); + } + } +private: + bool getH (const Matx33d &A, const Vec3d &e_prime, int smpl1, int smpl2, int smpl3, Matx33d &H) { + Vec3d p1(points[smpl1 ], points[smpl1+1], 1), p2(points[smpl2 ], points[smpl2+1], 1), p3(points[smpl3 ], points[smpl3+1], 1); + Vec3d P1(points[smpl1+2], points[smpl1+3], 1), P2(points[smpl2+2], points[smpl2+3], 1), P3(points[smpl3+2], points[smpl3+3], 1); + const Matx33d M (p1[0], p1[1], 1, p2[0], p2[1], 1, p3[0], p3[1], 1); + if (p1.cross(p2).dot(p3) * P1.cross(P2).dot(P3) < 0) return false; + // (x′i × e') + const Vec3d P1e = P1.cross(e_prime), P2e = P2.cross(e_prime), P3e = P3.cross(e_prime); + // x′i × (A xi))^T (x′i × e′) / ‖x′i×e′‖^2, + const Vec3d b (P1.cross(A * p1).dot(P1e) / (P1e[0]*P1e[0]+P1e[1]*P1e[1]+P1e[2]*P1e[2]), + P2.cross(A * p2).dot(P2e) / (P2e[0]*P2e[0]+P2e[1]*P2e[1]+P2e[2]*P2e[2]), + P3.cross(A * p3).dot(P3e) / (P3e[0]*P3e[0]+P3e[1]*P3e[1]+P3e[2]*P3e[2])); + + H = A - e_prime * (M.inv() * b).t(); + return true; + } + int getNonPlanarSupport (const Mat &F, const std::vector &pts, int num_pts) { + int non_rand_support = 0; + f_error->setModelParameters(F); + for (int pt = 0; pt < num_pts; pt++) + if (f_error->getError(pts[pt]) < f_threshold_sqr) + non_rand_support++; + return non_rand_support; + } + + int getNonPlanarSupport (const Mat &F, bool preemptive=false, int max_so_far=0) { + int non_rand_support = 0; + f_error->setModelParameters(F); + if (preemptive) { + const auto preemptive_thr = -num_h_outliers_eval + max_so_far; + for (int pt = 0; pt < num_h_outliers_eval; pt++) + if (f_error->getError(h_outliers_eval[pt]) < f_threshold_sqr) + non_rand_support++; + else if (non_rand_support - pt < preemptive_thr) + break; + } else { + for (int pt = 0; pt < num_h_outliers_eval; pt++) + if (f_error->getError(h_outliers_eval[pt]) < f_threshold_sqr) + non_rand_support++; + if (num_models_used_so_far < MAX_MODELS_TO_TEST && !true_K_given/*for K we know that recovered F cannot be degenerate*/) { + non_planar_supports[num_models_used_so_far++] = non_rand_support; + if (num_models_used_so_far == MAX_MODELS_TO_TEST) { + getLambda(non_planar_supports, 2.32, num_h_outliers_eval, 0, false, estimated_min_non_planar_support); + if (estimated_min_non_planar_support < 3) estimated_min_non_planar_support = 3; + } + } + } + return non_rand_support; } inline bool isModelValid(const Mat &F, const std::vector &sample) const override { return ep_deg.isModelValid(F, sample); } - bool recoverIfDegenerate (const std::vector &sample, const Mat &F_best, - Mat &non_degenerate_model, Score &non_degenerate_model_score) override { - non_degenerate_model_score = Score(); // set worst case - - // According to Two-view Geometry Estimation Unaffected by a Dominant Plane - // (http://cmp.felk.cvut.cz/~matas/papers/chum-degen-cvpr05.pdf) - // only 5 homographies enough to test - // triplets {1,2,3}, {4,5,6}, {1,2,7}, {4,5,7} and {3,6,7} - - // H = A - e' (M^-1 b)^T - // A = [e']_x F - // b_i = (x′i × (A xi))^T (x′i × e′)‖x′i×e′‖^−2, - // M is a 3×3 matrix with rows x^T_i - // epipole e' is left nullspace of F s.t. e′^T F=0, - - // find e', null space of F^T - Vec3d e_prime = F_best.col(0).cross(F_best.col(2)); - if (fabs(e_prime(0)) < 1e-10 && fabs(e_prime(1)) < 1e-10 && - fabs(e_prime(2)) < 1e-10) // if e' is zero - e_prime = F_best.col(1).cross(F_best.col(2)); - - const Matx33d A = Math::getSkewSymmetric(e_prime) * Matx33d(F_best); - - Vec3d xi_prime(0,0,1), xi(0,0,1), b; - Matx33d M(0,0,1,0,0,1,0,0,1); // last column of M is 1 - - bool is_model_degenerate = false; - for (const auto &h_i : h_sample) { // only 5 samples - for (int pt_i = 0; pt_i < 3; pt_i++) { - // find b and M - const int smpl = 4*sample[h_i[pt_i]]; - xi[0] = points[smpl]; - xi[1] = points[smpl+1]; - xi_prime[0] = points[smpl+2]; - xi_prime[1] = points[smpl+3]; - - // (x′i × e') - const Vec3d xprime_X_eprime = xi_prime.cross(e_prime); - - // (x′i × (A xi)) - const Vec3d xprime_X_Ax = xi_prime.cross(A * xi); - - // x′i × (A xi))^T (x′i × e′) / ‖x′i×e′‖^2, - b[pt_i] = xprime_X_Ax.dot(xprime_X_eprime) / - std::pow(norm(xprime_X_eprime), 2); - - // M from x^T - M(pt_i, 0) = xi[0]; - M(pt_i, 1) = xi[1]; - } - - // compute H - Matx33d H = A - e_prime * (M.inv() * b).t(); - - int inliers_out_plane = 0; - h_reproj_error->setModelParameters(Mat(H)); - - // find inliers from sample, points related to H, x' ~ Hx - for (int s = 0; s < sample_size; s++) - if (h_reproj_error->getError(sample[s]) > homography_threshold) - if (++inliers_out_plane > 2) - break; - - // if there are at least 5 points lying on plane then F is degenerate - if (inliers_out_plane <= 2) { - is_model_degenerate = true; - - // update homography by polishing on all inliers - int h_inls_cnt = 0; - const auto &h_errors = h_reproj_error->getErrors(Mat(H)); - for (int pt = 0; pt < points_size; pt++) - if (h_errors[pt] < homography_threshold) - h_inliers[h_inls_cnt++] = pt; - if (h_non_min_solver->estimate(h_inliers, h_inls_cnt, h_models, weights) != 0) - H = Matx33d(h_models[0]); - - Mat newF; - const Score newF_score = planeAndParallaxRANSAC(H, newF, h_errors); - if (newF_score.isBetter(non_degenerate_model_score)) { - // store non degenerate model - non_degenerate_model_score = newF_score; - newF.copyTo(non_degenerate_model); - } - } - } - return is_model_degenerate; - } - Ptr clone(int state) const override { - return makePtr(state, quality->clone(), *points_mat, - sample_size, homography_threshold); - } -private: - // RANSAC with plane-and-parallax to find new Fundamental matrix - Score planeAndParallaxRANSAC (const Matx33d &H, Mat &best_F, const std::vector &h_errors) { - int max_iters = 100; // with 95% confidence assume at least 17% of inliers - Score best_score; - for (int iters = 0; iters < max_iters; iters++) { - // draw two random points - int h_outlier1 = rng.uniform(0, points_size); - int h_outlier2 = rng.uniform(0, points_size); - while (h_outlier1 == h_outlier2) - h_outlier2 = rng.uniform(0, points_size); - - // find outliers of homography H - if (h_errors[h_outlier1] > homography_threshold && - h_errors[h_outlier2] > homography_threshold) { - - // do plane and parallax with outliers of H - // F = [(p1' x Hp1) x (p2' x Hp2)]_x H - const Matx33d F = Math::getSkewSymmetric( - (Vec3d(points[4*h_outlier1+2], points[4*h_outlier1+3], 1).cross // p1' - (H * Vec3d(points[4*h_outlier1 ], points[4*h_outlier1+1], 1))).cross // Hp1 - (Vec3d(points[4*h_outlier2+2], points[4*h_outlier2+3], 1).cross // p2' - (H * Vec3d(points[4*h_outlier2 ], points[4*h_outlier2+1], 1))) // Hp2 - ) * H; - - const Score score = quality->getScore(Mat(F)); - if (score.isBetter(best_score)) { - best_score = score; - best_F = Mat(F); - const double predicted_iters = log_conf / log(1 - std::pow - (static_cast(score.inlier_number) / points_size, 2)); - - if (! std::isinf(predicted_iters) && predicted_iters < max_iters) - max_iters = static_cast(predicted_iters); - } - } - } - return best_score; + bool isFDegenerate (int num_f_inliers_h_outliers) const { + if (num_models_used_so_far < MAX_MODELS_TO_TEST) + // the minimum number of non-planar support has not estimated yet -> use tentative + return num_f_inliers_h_outliers < std::min(TENT_MIN_NON_PLANAR_SUPP, (int)(0.1 * num_h_outliers_eval)); + return num_f_inliers_h_outliers < estimated_min_non_planar_support; } }; Ptr FundamentalDegeneracy::create (int state, const Ptr &quality_, - const Mat &points_, int sample_size_, double homography_threshold_) { + const Mat &points_, int sample_size_, int max_iters_plane_and_parallax, double homography_threshold_, + double f_inlier_thr_sqr, const Mat true_K1, const Mat true_K2) { return makePtr(state, quality_, points_, sample_size_, - homography_threshold_); + max_iters_plane_and_parallax, homography_threshold_, f_inlier_thr_sqr, true_K1, true_K2); } + class EssentialDegeneracyImpl : public EssentialDegeneracy { private: - const Mat * points_mat; - const int sample_size; const EpipolarGeometryDegeneracyImpl ep_deg; public: explicit EssentialDegeneracyImpl (const Mat &points, int sample_size_) : - points_mat(&points), sample_size(sample_size_), ep_deg (points, sample_size_) {} + ep_deg (points, sample_size_) {} inline bool isModelValid(const Mat &E, const std::vector &sample) const override { return ep_deg.isModelValid(E, sample); } - Ptr clone(int /*state*/) const override { - return makePtr(*points_mat, sample_size); - } }; Ptr EssentialDegeneracy::create (const Mat &points_, int sample_size_) { return makePtr(points_, sample_size_); } -}} +}} \ No newline at end of file diff --git a/modules/calib3d/src/usac/dls_solver.cpp b/modules/calib3d/src/usac/dls_solver.cpp index 8f109d51bf..43b0fc2919 100644 --- a/modules/calib3d/src/usac/dls_solver.cpp +++ b/modules/calib3d/src/usac/dls_solver.cpp @@ -46,29 +46,25 @@ #endif namespace cv { namespace usac { -// This is the estimator class for estimating a homography matrix between two images. A model estimation method and error calculation method are implemented class DLSPnPImpl : public DLSPnP { +#if defined(HAVE_LAPACK) || defined(HAVE_EIGEN) private: const Mat * points_mat, * calib_norm_points_mat; - const Matx33d * K_mat; -#if defined(HAVE_LAPACK) || defined(HAVE_EIGEN) - const Matx33d &K; + const Matx33d K; const float * const calib_norm_points, * const points; -#endif public: - explicit DLSPnPImpl (const Mat &points_, const Mat &calib_norm_points_, const Matx33d &K_) : - points_mat(&points_), calib_norm_points_mat(&calib_norm_points_), K_mat (&K_) -#if defined(HAVE_LAPACK) || defined(HAVE_EIGEN) - , K(K_), calib_norm_points((float*)calib_norm_points_.data), points((float*)points_.data) + explicit DLSPnPImpl (const Mat &points_, const Mat &calib_norm_points_, const Mat &K_) + : points_mat(&points_), calib_norm_points_mat(&calib_norm_points_), K(K_), + calib_norm_points((float*)calib_norm_points_mat->data), points((float*)points_mat->data) {} +#else +public: + explicit DLSPnPImpl (const Mat &, const Mat &, const Mat &) {} #endif - {} + // return minimal sample size required for non-minimal estimation. int getMinimumRequiredSampleSize() const override { return 3; } // return maximum number of possible solutions. int getMaxNumberOfSolutions () const override { return 27; } - Ptr clone () const override { - return makePtr(*points_mat, *calib_norm_points_mat, *K_mat); - } #if defined(HAVE_LAPACK) || defined(HAVE_EIGEN) int estimate(const std::vector &sample, int sample_number, std::vector &models_, const std::vector &/*weights_*/) const override { @@ -170,7 +166,6 @@ public: for (int i = 0; i < sample_number; i++) pts_random_shuffle[i] = i; randShuffle(pts_random_shuffle); - for (int i = 0; i < 27; i++) { // If the rotation solutions are real, treat this as a valid candidate // rotation. @@ -227,6 +222,12 @@ public: #endif } + int estimate (const std::vector &/*mask*/, std::vector &/*models*/, + const std::vector &/*weights*/) override { + return 0; + } + void enforceRankConstraint (bool /*enforce*/) override {} + protected: #if defined(HAVE_LAPACK) || defined(HAVE_EIGEN) const int indices[1968] = { @@ -871,7 +872,7 @@ protected: 2 * D[74] - 2 * D[78]); // s1^3 } }; -Ptr DLSPnP::create(const Mat &points_, const Mat &calib_norm_pts, const Matx33d &K) { +Ptr DLSPnP::create(const Mat &points_, const Mat &calib_norm_pts, const Mat &K) { return makePtr(points_, calib_norm_pts, K); } }} diff --git a/modules/calib3d/src/usac/essential_solver.cpp b/modules/calib3d/src/usac/essential_solver.cpp index 014cd36f40..e2b59a8c49 100644 --- a/modules/calib3d/src/usac/essential_solver.cpp +++ b/modules/calib3d/src/usac/essential_solver.cpp @@ -11,28 +11,25 @@ #endif namespace cv { namespace usac { -// Essential matrix solver: /* * H. Stewenius, C. Engels, and D. Nister. Recent developments on direct relative orientation. * ISPRS J. of Photogrammetry and Remote Sensing, 60:284,294, 2006 * http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.61.9329&rep=rep1&type=pdf +* +* D. Nister. An efficient solution to the five-point relative pose problem +* IEEE Transactions on Pattern Analysis and Machine Intelligence +* https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.86.8769&rep=rep1&type=pdf */ -class EssentialMinimalSolverStewenius5ptsImpl : public EssentialMinimalSolverStewenius5pts { +class EssentialMinimalSolver5ptsImpl : public EssentialMinimalSolver5pts { private: // Points must be calibrated K^-1 x const Mat * points_mat; -#if defined(HAVE_EIGEN) || defined(HAVE_LAPACK) const float * const pts; -#endif + const bool use_svd, is_nister; public: - explicit EssentialMinimalSolverStewenius5ptsImpl (const Mat &points_) : - points_mat(&points_) -#if defined(HAVE_EIGEN) || defined(HAVE_LAPACK) - , pts((float*)points_.data) -#endif - {} + explicit EssentialMinimalSolver5ptsImpl (const Mat &points_, bool use_svd_=false, bool is_nister_=false) : + points_mat(&points_), pts((float*)points_mat->data), use_svd(use_svd_), is_nister(is_nister_) {} -#if defined(HAVE_LAPACK) || defined(HAVE_EIGEN) int estimate (const std::vector &sample, std::vector &models) const override { // (1) Extract 4 null vectors from linear equations of epipolar constraint std::vector coefficients(45); // 5 pts=rows, 9 columns @@ -53,25 +50,35 @@ public: const int num_cols = 9, num_e_mat = 4; double ee[36]; // 9*4 - // eliminate linear equations - if (!Math::eliminateUpperTriangular(coefficients, 5, num_cols)) - return 0; - for (int i = 0; i < num_e_mat; i++) - for (int j = 5; j < num_cols; j++) - ee[num_cols * i + j] = (i + 5 == j) ? 1 : 0; - // use back-substitution - for (int e = 0; e < num_e_mat; e++) { - const int curr_e = num_cols * e; - // start from the last row - for (int i = 4; i >= 0; i--) { - const int row_i = i * num_cols; - double acc = 0; - for (int j = i + 1; j < num_cols; j++) - acc -= coefficients[row_i + j] * ee[curr_e + j]; - ee[curr_e + i] = acc / coefficients[row_i + i]; - // due to numerical errors return 0 solutions - if (std::isnan(ee[curr_e + i])) - return 0; + if (use_svd) { + Matx coeffs (&coefficients[0]); + Mat D, U, Vt; + SVDecomp(coeffs, D, U, Vt, SVD::FULL_UV + SVD::MODIFY_A); + const auto * const vt = (double *) Vt.data; + for (int i = 0; i < num_e_mat; i++) + for (int j = 0; j < num_cols; j++) + ee[i * num_cols + j] = vt[(8-i)*num_cols+j]; + } else { + // eliminate linear equations + if (!Math::eliminateUpperTriangular(coefficients, 5, num_cols)) + return 0; + for (int i = 0; i < num_e_mat; i++) + for (int j = 5; j < num_cols; j++) + ee[num_cols * i + j] = (i + 5 == j) ? 1 : 0; + // use back-substitution + for (int e = 0; e < num_e_mat; e++) { + const int curr_e = num_cols * e; + // start from the last row + for (int i = 4; i >= 0; i--) { + const int row_i = i * num_cols; + double acc = 0; + for (int j = i + 1; j < num_cols; j++) + acc -= coefficients[row_i + j] * ee[curr_e + j]; + ee[curr_e + i] = acc / coefficients[row_i + i]; + // due to numerical errors return 0 solutions + if (std::isnan(ee[curr_e + i])) + return 0; + } } } @@ -80,130 +87,215 @@ public: {null_space.col(0), null_space.col(3), null_space.col(6)}, {null_space.col(1), null_space.col(4), null_space.col(7)}, {null_space.col(2), null_space.col(5), null_space.col(8)}}; + Mat_ constraint_mat(10, 20); + Matx eet[3][3]; // (2) Use the rank constraint and the trace constraint to build ten third-order polynomial // equations in the three unknowns. The monomials are ordered in GrLex order and // represented in a 10×20 matrix, where each row corresponds to an equation and each column // corresponds to a monomial - Matx eet[3][3]; - for (int i = 0; i < 3; i++) - for (int j = 0; j < 3; j++) - // compute EE Transpose - // Shorthand for multiplying the Essential matrix with its transpose. - eet[i][j] = 2 * (multPolysDegOne(null_space_mat[i][0].val, null_space_mat[j][0].val) + - multPolysDegOne(null_space_mat[i][1].val, null_space_mat[j][1].val) + - multPolysDegOne(null_space_mat[i][2].val, null_space_mat[j][2].val)); + if (is_nister) { + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + // compute EE Transpose + // Shorthand for multiplying the Essential matrix with its transpose. + eet[i][j] = multPolysDegOne(null_space_mat[i][0].val, null_space_mat[j][0].val) + + multPolysDegOne(null_space_mat[i][1].val, null_space_mat[j][1].val) + + multPolysDegOne(null_space_mat[i][2].val, null_space_mat[j][2].val); - const Matx trace = eet[0][0] + eet[1][1] + eet[2][2]; - Mat_ constraint_mat(10, 20); - // Trace constraint - for (int i = 0; i < 3; i++) - for (int j = 0; j < 3; j++) - Mat(multPolysDegOneAndTwo(eet[i][0].val, null_space_mat[0][j].val) + - multPolysDegOneAndTwo(eet[i][1].val, null_space_mat[1][j].val) + - multPolysDegOneAndTwo(eet[i][2].val, null_space_mat[2][j].val) - - 0.5 * multPolysDegOneAndTwo(trace.val, null_space_mat[i][j].val)) - .copyTo(constraint_mat.row(3 * i + j)); + const Matx trace = 0.5*(eet[0][0] + eet[1][1] + eet[2][2]); + // Trace constraint + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + Mat(multPolysDegOneAndTwoNister(i == 0 ? (eet[i][0] - trace).val : eet[i][0].val, null_space_mat[0][j].val) + + multPolysDegOneAndTwoNister(i == 1 ? (eet[i][1] - trace).val : eet[i][1].val, null_space_mat[1][j].val) + + multPolysDegOneAndTwoNister(i == 2 ? (eet[i][2] - trace).val : eet[i][2].val, null_space_mat[2][j].val)).copyTo(constraint_mat.row(1+3 * j + i)); - // Rank = zero determinant constraint - Mat(multPolysDegOneAndTwo( - (multPolysDegOne(null_space_mat[0][1].val, null_space_mat[1][2].val) - - multPolysDegOne(null_space_mat[0][2].val, null_space_mat[1][1].val)).val, - null_space_mat[2][0].val) + + // Rank = zero determinant constraint + Mat(multPolysDegOneAndTwoNister( + (multPolysDegOne(null_space_mat[0][1].val, null_space_mat[1][2].val) - + multPolysDegOne(null_space_mat[0][2].val, null_space_mat[1][1].val)).val, + null_space_mat[2][0].val) + + multPolysDegOneAndTwoNister( + (multPolysDegOne(null_space_mat[0][2].val, null_space_mat[1][0].val) - + multPolysDegOne(null_space_mat[0][0].val, null_space_mat[1][2].val)).val, + null_space_mat[2][1].val) + + multPolysDegOneAndTwoNister( + (multPolysDegOne(null_space_mat[0][0].val, null_space_mat[1][1].val) - + multPolysDegOne(null_space_mat[0][1].val, null_space_mat[1][0].val)).val, + null_space_mat[2][2].val)).copyTo(constraint_mat.row(0)); + + Matx Acoef = constraint_mat.colRange(0, 10), + Bcoef = constraint_mat.colRange(10, 20), A_; + + if (!solve(Acoef, Bcoef, A_, DECOMP_LU)) return 0; + + double b[3 * 13]; + const auto * const a = A_.val; + for (int i = 0; i < 3; i++) { + const int r1_idx = i * 2 + 4, r2_idx = i * 2 + 5; // process from 4th row + for (int j = 0, r1_j = 0, r2_j = 0; j < 13; j++) + b[i*13+j] = ((j == 0 || j == 4 || j == 8) ? 0 : a[r1_idx*A_.cols+r1_j++]) - ((j == 3 || j == 7 || j == 12) ? 0 : a[r2_idx*A_.cols+r2_j++]); + } + + std::vector c(11), rs; + // filling coefficients of 10-degree polynomial satysfying zero-determinant constraint of essential matrix, ie., det(E) = 0 + // based on "An Efficient Solution to the Five-Point Relative Pose Problem" (David Nister) + // same as in five-point.cpp + c[10] = (b[0]*b[17]*b[34]+b[26]*b[4]*b[21]-b[26]*b[17]*b[8]-b[13]*b[4]*b[34]-b[0]*b[21]*b[30]+b[13]*b[30]*b[8]); + c[9] = (b[26]*b[4]*b[22]+b[14]*b[30]*b[8]+b[13]*b[31]*b[8]+b[1]*b[17]*b[34]-b[13]*b[5]*b[34]+b[26]*b[5]*b[21]-b[0]*b[21]*b[31]-b[26]*b[17]*b[9]-b[1]*b[21]*b[30]+b[27]*b[4]*b[21]+b[0]*b[17]*b[35]-b[0]*b[22]*b[30]+b[13]*b[30]*b[9]+b[0]*b[18]*b[34]-b[27]*b[17]*b[8]-b[14]*b[4]*b[34]-b[13]*b[4]*b[35]-b[26]*b[18]*b[8]); + c[8] = (b[14]*b[30]*b[9]+b[14]*b[31]*b[8]+b[13]*b[31]*b[9]-b[13]*b[4]*b[36]-b[13]*b[5]*b[35]+b[15]*b[30]*b[8]-b[13]*b[6]*b[34]+b[13]*b[30]*b[10]+b[13]*b[32]*b[8]-b[14]*b[4]*b[35]-b[14]*b[5]*b[34]+b[26]*b[4]*b[23]+b[26]*b[5]*b[22]+b[26]*b[6]*b[21]-b[26]*b[17]*b[10]-b[15]*b[4]*b[34]-b[26]*b[18]*b[9]-b[26]*b[19]*b[8]+b[27]*b[4]*b[22]+b[27]*b[5]*b[21]-b[27]*b[17]*b[9]-b[27]*b[18]*b[8]-b[1]*b[21]*b[31]-b[0]*b[23]*b[30]-b[0]*b[21]*b[32]+b[28]*b[4]*b[21]-b[28]*b[17]*b[8]+b[2]*b[17]*b[34]+b[0]*b[18]*b[35]-b[0]*b[22]*b[31]+b[0]*b[17]*b[36]+b[0]*b[19]*b[34]-b[1]*b[22]*b[30]+b[1]*b[18]*b[34]+b[1]*b[17]*b[35]-b[2]*b[21]*b[30]); + c[7] = (b[14]*b[30]*b[10]+b[14]*b[32]*b[8]-b[3]*b[21]*b[30]+b[3]*b[17]*b[34]+b[13]*b[32]*b[9]+b[13]*b[33]*b[8]-b[13]*b[4]*b[37]-b[13]*b[5]*b[36]+b[15]*b[30]*b[9]+b[15]*b[31]*b[8]-b[16]*b[4]*b[34]-b[13]*b[6]*b[35]-b[13]*b[7]*b[34]+b[13]*b[30]*b[11]+b[13]*b[31]*b[10]+b[14]*b[31]*b[9]-b[14]*b[4]*b[36]-b[14]*b[5]*b[35]-b[14]*b[6]*b[34]+b[16]*b[30]*b[8]-b[26]*b[20]*b[8]+b[26]*b[4]*b[24]+b[26]*b[5]*b[23]+b[26]*b[6]*b[22]+b[26]*b[7]*b[21]-b[26]*b[17]*b[11]-b[15]*b[4]*b[35]-b[15]*b[5]*b[34]-b[26]*b[18]*b[10]-b[26]*b[19]*b[9]+b[27]*b[4]*b[23]+b[27]*b[5]*b[22]+b[27]*b[6]*b[21]-b[27]*b[17]*b[10]-b[27]*b[18]*b[9]-b[27]*b[19]*b[8]+b[0]*b[17]*b[37]-b[0]*b[23]*b[31]-b[0]*b[24]*b[30]-b[0]*b[21]*b[33]-b[29]*b[17]*b[8]+b[28]*b[4]*b[22]+b[28]*b[5]*b[21]-b[28]*b[17]*b[9]-b[28]*b[18]*b[8]+b[29]*b[4]*b[21]+b[1]*b[19]*b[34]-b[2]*b[21]*b[31]+b[0]*b[20]*b[34]+b[0]*b[19]*b[35]+b[0]*b[18]*b[36]-b[0]*b[22]*b[32]-b[1]*b[23]*b[30]-b[1]*b[21]*b[32]+b[1]*b[18]*b[35]-b[1]*b[22]*b[31]-b[2]*b[22]*b[30]+b[2]*b[17]*b[35]+b[1]*b[17]*b[36]+b[2]*b[18]*b[34]); + c[6] = (-b[14]*b[6]*b[35]-b[14]*b[7]*b[34]-b[3]*b[22]*b[30]-b[3]*b[21]*b[31]+b[3]*b[17]*b[35]+b[3]*b[18]*b[34]+b[13]*b[32]*b[10]+b[13]*b[33]*b[9]-b[13]*b[4]*b[38]-b[13]*b[5]*b[37]-b[15]*b[6]*b[34]+b[15]*b[30]*b[10]+b[15]*b[32]*b[8]-b[16]*b[4]*b[35]-b[13]*b[6]*b[36]-b[13]*b[7]*b[35]+b[13]*b[31]*b[11]+b[13]*b[30]*b[12]+b[14]*b[32]*b[9]+b[14]*b[33]*b[8]-b[14]*b[4]*b[37]-b[14]*b[5]*b[36]+b[16]*b[30]*b[9]+b[16]*b[31]*b[8]-b[26]*b[20]*b[9]+b[26]*b[4]*b[25]+b[26]*b[5]*b[24]+b[26]*b[6]*b[23]+b[26]*b[7]*b[22]-b[26]*b[17]*b[12]+b[14]*b[30]*b[11]+b[14]*b[31]*b[10]+b[15]*b[31]*b[9]-b[15]*b[4]*b[36]-b[15]*b[5]*b[35]-b[26]*b[18]*b[11]-b[26]*b[19]*b[10]-b[27]*b[20]*b[8]+b[27]*b[4]*b[24]+b[27]*b[5]*b[23]+b[27]*b[6]*b[22]+b[27]*b[7]*b[21]-b[27]*b[17]*b[11]-b[27]*b[18]*b[10]-b[27]*b[19]*b[9]-b[16]*b[5]*b[34]-b[29]*b[17]*b[9]-b[29]*b[18]*b[8]+b[28]*b[4]*b[23]+b[28]*b[5]*b[22]+b[28]*b[6]*b[21]-b[28]*b[17]*b[10]-b[28]*b[18]*b[9]-b[28]*b[19]*b[8]+b[29]*b[4]*b[22]+b[29]*b[5]*b[21]-b[2]*b[23]*b[30]+b[2]*b[18]*b[35]-b[1]*b[22]*b[32]-b[2]*b[21]*b[32]+b[2]*b[19]*b[34]+b[0]*b[19]*b[36]-b[0]*b[22]*b[33]+b[0]*b[20]*b[35]-b[0]*b[23]*b[32]-b[0]*b[25]*b[30]+b[0]*b[17]*b[38]+b[0]*b[18]*b[37]-b[0]*b[24]*b[31]+b[1]*b[17]*b[37]-b[1]*b[23]*b[31]-b[1]*b[24]*b[30]-b[1]*b[21]*b[33]+b[1]*b[20]*b[34]+b[1]*b[19]*b[35]+b[1]*b[18]*b[36]+b[2]*b[17]*b[36]-b[2]*b[22]*b[31]); + c[5] = (-b[14]*b[6]*b[36]-b[14]*b[7]*b[35]+b[14]*b[31]*b[11]-b[3]*b[23]*b[30]-b[3]*b[21]*b[32]+b[3]*b[18]*b[35]-b[3]*b[22]*b[31]+b[3]*b[17]*b[36]+b[3]*b[19]*b[34]+b[13]*b[32]*b[11]+b[13]*b[33]*b[10]-b[13]*b[5]*b[38]-b[15]*b[6]*b[35]-b[15]*b[7]*b[34]+b[15]*b[30]*b[11]+b[15]*b[31]*b[10]+b[16]*b[31]*b[9]-b[13]*b[6]*b[37]-b[13]*b[7]*b[36]+b[13]*b[31]*b[12]+b[14]*b[32]*b[10]+b[14]*b[33]*b[9]-b[14]*b[4]*b[38]-b[14]*b[5]*b[37]-b[16]*b[6]*b[34]+b[16]*b[30]*b[10]+b[16]*b[32]*b[8]-b[26]*b[20]*b[10]+b[26]*b[5]*b[25]+b[26]*b[6]*b[24]+b[26]*b[7]*b[23]+b[14]*b[30]*b[12]+b[15]*b[32]*b[9]+b[15]*b[33]*b[8]-b[15]*b[4]*b[37]-b[15]*b[5]*b[36]+b[29]*b[5]*b[22]+b[29]*b[6]*b[21]-b[26]*b[18]*b[12]-b[26]*b[19]*b[11]-b[27]*b[20]*b[9]+b[27]*b[4]*b[25]+b[27]*b[5]*b[24]+b[27]*b[6]*b[23]+b[27]*b[7]*b[22]-b[27]*b[17]*b[12]-b[27]*b[18]*b[11]-b[27]*b[19]*b[10]-b[28]*b[20]*b[8]-b[16]*b[4]*b[36]-b[16]*b[5]*b[35]-b[29]*b[17]*b[10]-b[29]*b[18]*b[9]-b[29]*b[19]*b[8]+b[28]*b[4]*b[24]+b[28]*b[5]*b[23]+b[28]*b[6]*b[22]+b[28]*b[7]*b[21]-b[28]*b[17]*b[11]-b[28]*b[18]*b[10]-b[28]*b[19]*b[9]+b[29]*b[4]*b[23]-b[2]*b[22]*b[32]-b[2]*b[21]*b[33]-b[1]*b[24]*b[31]+b[0]*b[18]*b[38]-b[0]*b[24]*b[32]+b[0]*b[19]*b[37]+b[0]*b[20]*b[36]-b[0]*b[25]*b[31]-b[0]*b[23]*b[33]+b[1]*b[19]*b[36]-b[1]*b[22]*b[33]+b[1]*b[20]*b[35]+b[2]*b[19]*b[35]-b[2]*b[24]*b[30]-b[2]*b[23]*b[31]+b[2]*b[20]*b[34]+b[2]*b[17]*b[37]-b[1]*b[25]*b[30]+b[1]*b[18]*b[37]+b[1]*b[17]*b[38]-b[1]*b[23]*b[32]+b[2]*b[18]*b[36]); + c[4] = (-b[14]*b[6]*b[37]-b[14]*b[7]*b[36]+b[14]*b[31]*b[12]+b[3]*b[17]*b[37]-b[3]*b[23]*b[31]-b[3]*b[24]*b[30]-b[3]*b[21]*b[33]+b[3]*b[20]*b[34]+b[3]*b[19]*b[35]+b[3]*b[18]*b[36]-b[3]*b[22]*b[32]+b[13]*b[32]*b[12]+b[13]*b[33]*b[11]-b[15]*b[6]*b[36]-b[15]*b[7]*b[35]+b[15]*b[31]*b[11]+b[15]*b[30]*b[12]+b[16]*b[32]*b[9]+b[16]*b[33]*b[8]-b[13]*b[6]*b[38]-b[13]*b[7]*b[37]+b[14]*b[32]*b[11]+b[14]*b[33]*b[10]-b[14]*b[5]*b[38]-b[16]*b[6]*b[35]-b[16]*b[7]*b[34]+b[16]*b[30]*b[11]+b[16]*b[31]*b[10]-b[26]*b[19]*b[12]-b[26]*b[20]*b[11]+b[26]*b[6]*b[25]+b[26]*b[7]*b[24]+b[15]*b[32]*b[10]+b[15]*b[33]*b[9]-b[15]*b[4]*b[38]-b[15]*b[5]*b[37]+b[29]*b[5]*b[23]+b[29]*b[6]*b[22]+b[29]*b[7]*b[21]-b[27]*b[20]*b[10]+b[27]*b[5]*b[25]+b[27]*b[6]*b[24]+b[27]*b[7]*b[23]-b[27]*b[18]*b[12]-b[27]*b[19]*b[11]-b[28]*b[20]*b[9]-b[16]*b[4]*b[37]-b[16]*b[5]*b[36]+b[0]*b[19]*b[38]-b[0]*b[24]*b[33]+b[0]*b[20]*b[37]-b[29]*b[17]*b[11]-b[29]*b[18]*b[10]-b[29]*b[19]*b[9]+b[28]*b[4]*b[25]+b[28]*b[5]*b[24]+b[28]*b[6]*b[23]+b[28]*b[7]*b[22]-b[28]*b[17]*b[12]-b[28]*b[18]*b[11]-b[28]*b[19]*b[10]-b[29]*b[20]*b[8]+b[29]*b[4]*b[24]+b[2]*b[18]*b[37]-b[0]*b[25]*b[32]+b[1]*b[18]*b[38]-b[1]*b[24]*b[32]+b[1]*b[19]*b[37]+b[1]*b[20]*b[36]-b[1]*b[25]*b[31]+b[2]*b[17]*b[38]+b[2]*b[19]*b[36]-b[2]*b[24]*b[31]-b[2]*b[22]*b[33]-b[2]*b[23]*b[32]+b[2]*b[20]*b[35]-b[1]*b[23]*b[33]-b[2]*b[25]*b[30]); + c[3] = (-b[14]*b[6]*b[38]-b[14]*b[7]*b[37]+b[3]*b[19]*b[36]-b[3]*b[22]*b[33]+b[3]*b[20]*b[35]-b[3]*b[23]*b[32]-b[3]*b[25]*b[30]+b[3]*b[17]*b[38]+b[3]*b[18]*b[37]-b[3]*b[24]*b[31]-b[15]*b[6]*b[37]-b[15]*b[7]*b[36]+b[15]*b[31]*b[12]+b[16]*b[32]*b[10]+b[16]*b[33]*b[9]+b[13]*b[33]*b[12]-b[13]*b[7]*b[38]+b[14]*b[32]*b[12]+b[14]*b[33]*b[11]-b[16]*b[6]*b[36]-b[16]*b[7]*b[35]+b[16]*b[31]*b[11]+b[16]*b[30]*b[12]+b[15]*b[32]*b[11]+b[15]*b[33]*b[10]-b[15]*b[5]*b[38]+b[29]*b[5]*b[24]+b[29]*b[6]*b[23]-b[26]*b[20]*b[12]+b[26]*b[7]*b[25]-b[27]*b[19]*b[12]-b[27]*b[20]*b[11]+b[27]*b[6]*b[25]+b[27]*b[7]*b[24]-b[28]*b[20]*b[10]-b[16]*b[4]*b[38]-b[16]*b[5]*b[37]+b[29]*b[7]*b[22]-b[29]*b[17]*b[12]-b[29]*b[18]*b[11]-b[29]*b[19]*b[10]+b[28]*b[5]*b[25]+b[28]*b[6]*b[24]+b[28]*b[7]*b[23]-b[28]*b[18]*b[12]-b[28]*b[19]*b[11]-b[29]*b[20]*b[9]+b[29]*b[4]*b[25]-b[2]*b[24]*b[32]+b[0]*b[20]*b[38]-b[0]*b[25]*b[33]+b[1]*b[19]*b[38]-b[1]*b[24]*b[33]+b[1]*b[20]*b[37]-b[2]*b[25]*b[31]+b[2]*b[20]*b[36]-b[1]*b[25]*b[32]+b[2]*b[19]*b[37]+b[2]*b[18]*b[38]-b[2]*b[23]*b[33]); + c[2] = (b[3]*b[18]*b[38]-b[3]*b[24]*b[32]+b[3]*b[19]*b[37]+b[3]*b[20]*b[36]-b[3]*b[25]*b[31]-b[3]*b[23]*b[33]-b[15]*b[6]*b[38]-b[15]*b[7]*b[37]+b[16]*b[32]*b[11]+b[16]*b[33]*b[10]-b[16]*b[5]*b[38]-b[16]*b[6]*b[37]-b[16]*b[7]*b[36]+b[16]*b[31]*b[12]+b[14]*b[33]*b[12]-b[14]*b[7]*b[38]+b[15]*b[32]*b[12]+b[15]*b[33]*b[11]+b[29]*b[5]*b[25]+b[29]*b[6]*b[24]-b[27]*b[20]*b[12]+b[27]*b[7]*b[25]-b[28]*b[19]*b[12]-b[28]*b[20]*b[11]+b[29]*b[7]*b[23]-b[29]*b[18]*b[12]-b[29]*b[19]*b[11]+b[28]*b[6]*b[25]+b[28]*b[7]*b[24]-b[29]*b[20]*b[10]+b[2]*b[19]*b[38]-b[1]*b[25]*b[33]+b[2]*b[20]*b[37]-b[2]*b[24]*b[33]-b[2]*b[25]*b[32]+b[1]*b[20]*b[38]); + c[1] = (b[29]*b[7]*b[24]-b[29]*b[20]*b[11]+b[2]*b[20]*b[38]-b[2]*b[25]*b[33]-b[28]*b[20]*b[12]+b[28]*b[7]*b[25]-b[29]*b[19]*b[12]-b[3]*b[24]*b[33]+b[15]*b[33]*b[12]+b[3]*b[19]*b[38]-b[16]*b[6]*b[38]+b[3]*b[20]*b[37]+b[16]*b[32]*b[12]+b[29]*b[6]*b[25]-b[16]*b[7]*b[37]-b[3]*b[25]*b[32]-b[15]*b[7]*b[38]+b[16]*b[33]*b[11]); + c[0] = -b[29]*b[20]*b[12]+b[29]*b[7]*b[25]+b[16]*b[33]*b[12]-b[16]*b[7]*b[38]+b[3]*b[20]*b[38]-b[3]*b[25]*b[33]; + + const auto poly_solver = SolverPoly::create(); + const int num_roots = poly_solver->getRealRoots(c, rs); + + models = std::vector(); models.reserve(num_roots); + for (int i = 0; i < num_roots; i++) { + const double z1 = rs[i], z2 = z1*z1, z3 = z2*z1, z4 = z3*z1; + double bz[9], norm_bz = 0; + for (int j = 0; j < 3; j++) { + double * const br = b + j * 13, * Bz = bz + 3*j; + Bz[0] = br[0] * z3 + br[1] * z2 + br[2] * z1 + br[3]; + Bz[1] = br[4] * z3 + br[5] * z2 + br[6] * z1 + br[7]; + Bz[2] = br[8] * z4 + br[9] * z3 + br[10] * z2 + br[11] * z1 + br[12]; + norm_bz += Bz[0]*Bz[0] + Bz[1]*Bz[1] + Bz[2]*Bz[2]; + } + + Matx33d Bz(bz); + // Bz is rank 2, matrix, so epipole is its null-vector + Vec3d xy1 = Utils::getRightEpipole(Mat(Bz * (1/sqrt(norm_bz)))); + + if (fabs(xy1(2)) < 1e-10) continue; + Mat_ E(3,3); + double * e_arr = (double *)E.data, x = xy1(0) / xy1(2), y = xy1(1) / xy1(2); + for (int e_i = 0; e_i < 9; e_i++) + e_arr[e_i] = ee[e_i] * x + ee[9+e_i] * y + ee[18+e_i]*z1 + ee[27+e_i]; + models.emplace_back(E); + } + } else { +#if defined(HAVE_EIGEN) || defined(HAVE_LAPACK) + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + // compute EE Transpose + // Shorthand for multiplying the Essential matrix with its transpose. + eet[i][j] = 2 * (multPolysDegOne(null_space_mat[i][0].val, null_space_mat[j][0].val) + + multPolysDegOne(null_space_mat[i][1].val, null_space_mat[j][1].val) + + multPolysDegOne(null_space_mat[i][2].val, null_space_mat[j][2].val)); + + const Matx trace = eet[0][0] + eet[1][1] + eet[2][2]; + // Trace constraint + for (int i = 0; i < 3; i++) + for (int j = 0; j < 3; j++) + Mat(multPolysDegOneAndTwo(eet[i][0].val, null_space_mat[0][j].val) + + multPolysDegOneAndTwo(eet[i][1].val, null_space_mat[1][j].val) + + multPolysDegOneAndTwo(eet[i][2].val, null_space_mat[2][j].val) - + 0.5 * multPolysDegOneAndTwo(trace.val, null_space_mat[i][j].val)) + .copyTo(constraint_mat.row(3 * i + j)); + + // Rank = zero determinant constraint + Mat(multPolysDegOneAndTwo( + (multPolysDegOne(null_space_mat[0][1].val, null_space_mat[1][2].val) - + multPolysDegOne(null_space_mat[0][2].val, null_space_mat[1][1].val)).val, + null_space_mat[2][0].val) + multPolysDegOneAndTwo( (multPolysDegOne(null_space_mat[0][2].val, null_space_mat[1][0].val) - multPolysDegOne(null_space_mat[0][0].val, null_space_mat[1][2].val)).val, - null_space_mat[2][1].val) + + null_space_mat[2][1].val) + multPolysDegOneAndTwo( (multPolysDegOne(null_space_mat[0][0].val, null_space_mat[1][1].val) - multPolysDegOne(null_space_mat[0][1].val, null_space_mat[1][0].val)).val, - null_space_mat[2][2].val)).copyTo(constraint_mat.row(9)); + null_space_mat[2][2].val)).copyTo(constraint_mat.row(9)); #ifdef HAVE_EIGEN - const Eigen::Matrix constraint_mat_eig((double *) constraint_mat.data); - // (3) Compute the Gröbner basis. This turns out to be as simple as performing a - // Gauss-Jordan elimination on the 10×20 matrix - const Eigen::Matrix eliminated_mat_eig = constraint_mat_eig.block<10, 10>(0, 0) - .fullPivLu().solve(constraint_mat_eig.block<10, 10>(0, 10)); + const Eigen::Matrix constraint_mat_eig((double *) constraint_mat.data); + // (3) Compute the Gröbner basis. This turns out to be as simple as performing a + // Gauss-Jordan elimination on the 10×20 matrix + const Eigen::Matrix eliminated_mat_eig = constraint_mat_eig.block<10, 10>(0, 0) + .fullPivLu().solve(constraint_mat_eig.block<10, 10>(0, 10)); - // (4) Compute the 10×10 action matrix for multiplication by one of the un-knowns. - // This is a simple matter of extracting the correct elements fromthe eliminated - // 10×20 matrix and organising them to form the action matrix. - Eigen::Matrix action_mat_eig = Eigen::Matrix::Zero(); - action_mat_eig.block<3, 10>(0, 0) = eliminated_mat_eig.block<3, 10>(0, 0); - action_mat_eig.block<2, 10>(3, 0) = eliminated_mat_eig.block<2, 10>(4, 0); - action_mat_eig.row(5) = eliminated_mat_eig.row(7); - action_mat_eig(6, 0) = -1.0; - action_mat_eig(7, 1) = -1.0; - action_mat_eig(8, 3) = -1.0; - action_mat_eig(9, 6) = -1.0; + // (4) Compute the 10×10 action matrix for multiplication by one of the unknowns. + // This is a simple matter of extracting the correct elements from the eliminated + // 10×20 matrix and organising them to form the action matrix. + Eigen::Matrix action_mat_eig = Eigen::Matrix::Zero(); + action_mat_eig.block<3, 10>(0, 0) = eliminated_mat_eig.block<3, 10>(0, 0); + action_mat_eig.block<2, 10>(3, 0) = eliminated_mat_eig.block<2, 10>(4, 0); + action_mat_eig.row(5) = eliminated_mat_eig.row(7); + action_mat_eig(6, 0) = -1.0; + action_mat_eig(7, 1) = -1.0; + action_mat_eig(8, 3) = -1.0; + action_mat_eig(9, 6) = -1.0; - // (5) Compute the left eigenvectors of the action matrix - Eigen::EigenSolver> eigensolver(action_mat_eig); - const Eigen::VectorXcd &eigenvalues = eigensolver.eigenvalues(); - const auto * const eig_vecs_ = (double *) eigensolver.eigenvectors().real().data(); + // (5) Compute the left eigenvectors of the action matrix + Eigen::EigenSolver> eigensolver(action_mat_eig); + const Eigen::VectorXcd &eigenvalues = eigensolver.eigenvalues(); + const auto * const eig_vecs_ = (double *) eigensolver.eigenvectors().real().data(); #else - Matx A = constraint_mat.colRange(0, 10), - B = constraint_mat.colRange(10, 20), eliminated_mat; - if (!solve(A, B, eliminated_mat, DECOMP_LU)) return 0; + Matx A = constraint_mat.colRange(0, 10), + B = constraint_mat.colRange(10, 20), eliminated_mat; + if (!solve(A, B, eliminated_mat, DECOMP_LU)) return 0; - Mat eliminated_mat_dyn = Mat(eliminated_mat); - Mat action_mat = Mat_::zeros(10, 10); - eliminated_mat_dyn.rowRange(0,3).copyTo(action_mat.rowRange(0,3)); - eliminated_mat_dyn.rowRange(4,6).copyTo(action_mat.rowRange(3,5)); - eliminated_mat_dyn.row(7).copyTo(action_mat.row(5)); - auto * action_mat_data = (double *) action_mat.data; - action_mat_data[60] = -1.0; // 6 row, 0 col - action_mat_data[71] = -1.0; // 7 row, 1 col - action_mat_data[83] = -1.0; // 8 row, 3 col - action_mat_data[96] = -1.0; // 9 row, 6 col + Mat eliminated_mat_dyn = Mat(eliminated_mat); + Mat action_mat = Mat_::zeros(10, 10); + eliminated_mat_dyn.rowRange(0,3).copyTo(action_mat.rowRange(0,3)); + eliminated_mat_dyn.rowRange(4,6).copyTo(action_mat.rowRange(3,5)); + eliminated_mat_dyn.row(7).copyTo(action_mat.row(5)); + auto * action_mat_data = (double *) action_mat.data; + action_mat_data[60] = -1.0; // 6 row, 0 col + action_mat_data[71] = -1.0; // 7 row, 1 col + action_mat_data[83] = -1.0; // 8 row, 3 col + action_mat_data[96] = -1.0; // 9 row, 6 col - int mat_order = 10, info, lda = 10, ldvl = 10, ldvr = 1, lwork = 100; - double wr[10], wi[10] = {0}, eig_vecs[100], work[100]; // 10 = mat_order, 100 = lwork - char jobvl = 'V', jobvr = 'N'; // only left eigen vectors are computed - OCV_LAPACK_FUNC(dgeev)(&jobvl, &jobvr, &mat_order, action_mat_data, &lda, wr, wi, eig_vecs, &ldvl, - nullptr, &ldvr, work, &lwork, &info); - if (info != 0) return 0; + int mat_order = 10, info, lda = 10, ldvl = 10, ldvr = 1, lwork = 100; + double wr[10], wi[10] = {0}, eig_vecs[100], work[100]; // 10 = mat_order, 100 = lwork + char jobvl = 'V', jobvr = 'N'; // only left eigen vectors are computed + OCV_LAPACK_FUNC(dgeev)(&jobvl, &jobvr, &mat_order, action_mat_data, &lda, wr, wi, eig_vecs, &ldvl, + nullptr, &ldvr, work, &lwork, &info); + if (info != 0) return 0; #endif + models = std::vector(); models.reserve(10); - models = std::vector(); models.reserve(10); - - // Read off the values for the three unknowns at all the solution points and - // back-substitute to obtain the solutions for the essential matrix. - for (int i = 0; i < 10; i++) - // process only real solutions + // Read off the values for the three unknowns at all the solution points and + // back-substitute to obtain the solutions for the essential matrix. + for (int i = 0; i < 10; i++) + // process only real solutions #ifdef HAVE_EIGEN - if (eigenvalues(i).imag() == 0) { - Mat_ model(3, 3); - auto * model_data = (double *) model.data; - const int eig_i = 20 * i + 12; // eigen stores imaginary values too - for (int j = 0; j < 9; j++) - model_data[j] = ee[j ] * eig_vecs_[eig_i ] + ee[j+9 ] * eig_vecs_[eig_i+2] + - ee[j+18] * eig_vecs_[eig_i+4] + ee[j+27] * eig_vecs_[eig_i+6]; + if (eigenvalues(i).imag() == 0) { + Mat_ model(3, 3); + auto * model_data = (double *) model.data; + const int eig_i = 20 * i + 12; // eigen stores imaginary values too + for (int j = 0; j < 9; j++) + model_data[j] = ee[j ] * eig_vecs_[eig_i ] + ee[j+9 ] * eig_vecs_[eig_i+2] + + ee[j+18] * eig_vecs_[eig_i+4] + ee[j+27] * eig_vecs_[eig_i+6]; #else - if (wi[i] == 0) { - Mat_ model (3,3); - auto * model_data = (double *) model.data; - const int eig_i = 10 * i + 6; - for (int j = 0; j < 9; j++) - model_data[j] = ee[j ]*eig_vecs[eig_i ] + ee[j+9 ]*eig_vecs[eig_i+1] + - ee[j+18]*eig_vecs[eig_i+2] + ee[j+27]*eig_vecs[eig_i+3]; + if (wi[i] == 0) { + Mat_ model (3,3); + auto * model_data = (double *) model.data; + const int eig_i = 10 * i + 6; + for (int j = 0; j < 9; j++) + model_data[j] = ee[j ]*eig_vecs[eig_i ] + ee[j+9 ]*eig_vecs[eig_i+1] + + ee[j+18]*eig_vecs[eig_i+2] + ee[j+27]*eig_vecs[eig_i+3]; #endif - models.emplace_back(model); - } + models.emplace_back(model); + } +#else + CV_Error(cv::Error::StsNotImplemented, "To run essential matrix estimation of Stewenius method you need to have either Eigen or LAPACK installed! Or switch to Nister algorithm"); + return 0; +#endif + } return static_cast(models.size()); -#else - int estimate (const std::vector &/*sample*/, std::vector &/*models*/) const override { - CV_Error(cv::Error::StsNotImplemented, "To use essential matrix solver LAPACK or Eigen has to be installed!"); -#endif } // number of possible solutions is 0,2,4,6,8,10 int getMaxNumberOfSolutions () const override { return 10; } int getSampleSize() const override { return 5; } - Ptr clone () const override { - return makePtr(*points_mat); - } private: /* * Multiply two polynomials of degree one with unknowns x y z @@ -236,105 +328,19 @@ private: p[5]*q[3]+p[8]*q[2], p[6]*q[3]+p[9]*q[0], p[7]*q[3]+p[9]*q[1], p[8]*q[3]+p[9]*q[2], p[9]*q[3]}); } -}; -Ptr EssentialMinimalSolverStewenius5pts::create - (const Mat &points_) { - return makePtr(points_); -} - -class EssentialNonMinimalSolverImpl : public EssentialNonMinimalSolver { -private: - const Mat * points_mat; - const float * const points; -public: - /* - * Input calibrated points K^-1 x. - * Linear 8 points algorithm is used for estimation. - */ - explicit EssentialNonMinimalSolverImpl (const Mat &points_) : - points_mat(&points_), points ((float *) points_.data) {} - - int estimate (const std::vector &sample, int sample_size, std::vector - &models, const std::vector &weights) const override { - if (sample_size < getMinimumRequiredSampleSize()) - return 0; - - // ------- 8 points algorithm with Eigen and covariance matrix -------------- - double a[9] = {0, 0, 0, 0, 0, 0, 0, 0, 1}; - double AtA[81] = {0}; // 9x9 - - if (weights.empty()) { - for (int i = 0; i < sample_size; i++) { - const int pidx = 4*sample[i]; - const double x1 = points[pidx ], y1 = points[pidx+1], - x2 = points[pidx+2], y2 = points[pidx+3]; - a[0] = x2*x1; - a[1] = x2*y1; - a[2] = x2; - a[3] = y2*x1; - a[4] = y2*y1; - a[5] = y2; - a[6] = x1; - a[7] = y1; - - // calculate covariance for eigen - for (int row = 0; row < 9; row++) - for (int col = row; col < 9; col++) - AtA[row*9+col] += a[row]*a[col]; - } - } else { - for (int i = 0; i < sample_size; i++) { - const int smpl = 4*sample[i]; - const double weight = weights[i]; - const double x1 = points[smpl ], y1 = points[smpl+1], - x2 = points[smpl+2], y2 = points[smpl+3]; - const double weight_times_x2 = weight * x2, - weight_times_y2 = weight * y2; - - a[0] = weight_times_x2 * x1; - a[1] = weight_times_x2 * y1; - a[2] = weight_times_x2; - a[3] = weight_times_y2 * x1; - a[4] = weight_times_y2 * y1; - a[5] = weight_times_y2; - a[6] = weight * x1; - a[7] = weight * y1; - a[8] = weight; - - // calculate covariance for eigen - for (int row = 0; row < 9; row++) - for (int col = row; col < 9; col++) - AtA[row*9+col] += a[row]*a[col]; - } - } - - // copy symmetric part of covariance matrix - for (int j = 1; j < 9; j++) - for (int z = 0; z < j; z++) - AtA[j*9+z] = AtA[z*9+j]; - -#ifdef HAVE_EIGEN - models = std::vector{ Mat_(3,3) }; - const Eigen::JacobiSVD> svd((Eigen::Matrix(AtA)), - Eigen::ComputeFullV); - // extract the last nullspace - Eigen::Map>((double *)models[0].data) = svd.matrixV().col(8); -#else - Matx AtA_(AtA), U, Vt; - Vec W; - SVD::compute(AtA_, W, U, Vt, SVD::FULL_UV + SVD::MODIFY_A); - models = std::vector { Mat_(3, 3, Vt.val + 72 /*=8*9*/) }; -#endif - FundamentalDegeneracy::recoverRank(models[0], false /*E*/); - return 1; - } - int getMinimumRequiredSampleSize() const override { return 8; } - int getMaxNumberOfSolutions () const override { return 1; } - Ptr clone () const override { - return makePtr(*points_mat); + static inline Matx multPolysDegOneAndTwoNister(const double * const p, + const double * const q) { + // permutation {0, 3, 1, 2, 4, 10, 6, 12, 5, 11, 7, 13, 16, 8, 14, 17, 9, 15, 18, 19}; + return Matx + ({p[0]*q[0], p[2]*q[1], p[0]*q[1]+p[1]*q[0], p[1]*q[1]+p[2]*q[0], p[0]*q[2]+p[3]*q[0], + p[0]*q[3]+p[6]*q[0], p[2]*q[2]+p[4]*q[1], p[2]*q[3]+p[7]*q[1], p[1]*q[2]+p[3]*q[1]+p[4]*q[0], + p[1]*q[3]+p[6]*q[1]+p[7]*q[0], p[3]*q[2]+p[5]*q[0], p[3]*q[3]+p[6]*q[2]+p[8]*q[0], + p[6]*q[3]+p[9]*q[0], p[4]*q[2]+p[5]*q[1], p[4]*q[3]+p[7]*q[2]+p[8]*q[1], p[7]*q[3]+p[9]*q[1], + p[5]*q[2], p[5]*q[3]+p[8]*q[2], p[8]*q[3]+p[9]*q[2], p[9]*q[3]}); } }; -Ptr EssentialNonMinimalSolver::create (const Mat &points_) { - return makePtr(points_); +Ptr EssentialMinimalSolver5pts::create + (const Mat &points_, bool use_svd, bool is_nister) { + return makePtr(points_, use_svd, is_nister); } }} \ No newline at end of file diff --git a/modules/calib3d/src/usac/estimator.cpp b/modules/calib3d/src/usac/estimator.cpp index c5b783b655..f68ebf4cef 100644 --- a/modules/calib3d/src/usac/estimator.cpp +++ b/modules/calib3d/src/usac/estimator.cpp @@ -36,10 +36,7 @@ public: int getNonMinimalSampleSize () const override { return non_min_solver->getMinimumRequiredSampleSize(); } - Ptr clone() const override { - return makePtr(min_solver->clone(), non_min_solver->clone(), - degeneracy->clone(0 /*we don't need state here*/)); - } + void enforceRankConstraint (bool /*enforce*/) override {} }; Ptr HomographyEstimator::create (const Ptr &min_solver_, const Ptr &non_min_solver_, const Ptr °eneracy_) { @@ -80,13 +77,12 @@ public: int getNonMinimalSampleSize () const override { return non_min_solver->getMinimumRequiredSampleSize(); } + void enforceRankConstraint (bool enforce) override { + non_min_solver->enforceRankConstraint(enforce); + } int getMaxNumSolutionsNonMinimal () const override { return non_min_solver->getMaxNumberOfSolutions(); } - Ptr clone() const override { - return makePtr(min_solver->clone(), non_min_solver->clone(), - degeneracy->clone(0)); - } }; Ptr FundamentalEstimator::create (const Ptr &min_solver_, const Ptr &non_min_solver_, const Ptr °eneracy_) { @@ -106,7 +102,7 @@ public: inline int estimateModels(const std::vector &sample, std::vector &models) const override { - std::vector E; + std::vector E; const int models_count = min_solver->estimate(sample, E); int valid_models_count = 0; for (int i = 0; i < models_count; i++) @@ -115,6 +111,10 @@ public: return valid_models_count; } + int estimateModelNonMinimalSample (const Mat &model, const std::vector &sample, int sample_size, std::vector + &models, const std::vector &weights) const override { + return non_min_solver->estimate(model, sample, sample_size, models, weights); + } int estimateModelNonMinimalSample(const std::vector &sample, int sample_size, std::vector &models, const std::vector &weights) const override { return non_min_solver->estimate(sample, sample_size, models, weights); @@ -128,13 +128,12 @@ public: int getNonMinimalSampleSize () const override { return non_min_solver->getMinimumRequiredSampleSize(); } + void enforceRankConstraint (bool enforce) override { + non_min_solver->enforceRankConstraint(enforce); + } int getMaxNumSolutionsNonMinimal () const override { return non_min_solver->getMaxNumberOfSolutions(); } - Ptr clone() const override { - return makePtr(min_solver->clone(), non_min_solver->clone(), - degeneracy->clone(0)); - } }; Ptr EssentialEstimator::create (const Ptr &min_solver_, const Ptr &non_min_solver_, const Ptr °eneracy_) { @@ -170,9 +169,7 @@ public: int getMaxNumSolutionsNonMinimal () const override { return non_min_solver->getMaxNumberOfSolutions(); } - Ptr clone() const override { - return makePtr(min_solver->clone(), non_min_solver->clone()); - } + void enforceRankConstraint (bool /*enforce*/) override {} }; Ptr AffineEstimator::create (const Ptr &min_solver_, const Ptr &non_min_solver_) { @@ -208,9 +205,7 @@ public: int getMaxNumSolutionsNonMinimal () const override { return non_min_solver->getMaxNumberOfSolutions(); } - Ptr clone() const override { - return makePtr(min_solver->clone(), non_min_solver->clone()); - } + void enforceRankConstraint (bool /*enforce*/) override {} }; Ptr PnPEstimator::create (const Ptr &min_solver_, const Ptr &non_min_solver_) { @@ -253,9 +248,9 @@ public: minv21=(float)minv[3]; minv22=(float)minv[4]; minv23=(float)minv[5]; minv31=(float)minv[6]; minv32=(float)minv[7]; minv33=(float)minv[8]; } - inline float getError (int point_idx) const override { - const int smpl = 4*point_idx; - const float x1=points[smpl], y1=points[smpl+1], x2=points[smpl+2], y2=points[smpl+3]; + inline float getError (int idx) const override { + idx *= 4; + const float x1=points[idx], y1=points[idx+1], x2=points[idx+2], y2=points[idx+3]; const float est_z2 = 1 / (m31 * x1 + m32 * y1 + m33), dx2 = x2 - (m11 * x1 + m12 * y1 + m13) * est_z2, dy2 = y2 - (m21 * x1 + m22 * y1 + m23) * est_z2; @@ -279,9 +274,6 @@ public: } return errors; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; Ptr ReprojectionErrorSymmetric::create(const Mat &points) { @@ -314,9 +306,9 @@ public: m21=static_cast(m[3]); m22=static_cast(m[4]); m23=static_cast(m[5]); m31=static_cast(m[6]); m32=static_cast(m[7]); m33=static_cast(m[8]); } - inline float getError (int point_idx) const override { - const int smpl = 4*point_idx; - const float x1 = points[smpl], y1 = points[smpl+1], x2 = points[smpl+2], y2 = points[smpl+3]; + inline float getError (int idx) const override { + idx *= 4; + const float x1 = points[idx], y1 = points[idx+1], x2 = points[idx+2], y2 = points[idx+3]; const float est_z2 = 1 / (m31 * x1 + m32 * y1 + m33), dx2 = x2 - (m11 * x1 + m12 * y1 + m13) * est_z2, dy2 = y2 - (m21 * x1 + m22 * y1 + m23) * est_z2; @@ -334,9 +326,6 @@ public: } return errors; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; Ptr ReprojectionErrorForward::create(const Mat &points) { @@ -379,9 +368,9 @@ public: * [ F(3,1) F(3,2) F(3,3) ] [ 1 ] * */ - inline float getError (int point_idx) const override { - const int smpl = 4*point_idx; - const float x1=points[smpl], y1=points[smpl+1], x2=points[smpl+2], y2=points[smpl+3]; + inline float getError (int idx) const override { + idx *= 4; + const float x1=points[idx], y1=points[idx+1], x2=points[idx+2], y2=points[idx+3]; const float F_pt1_x = m11 * x1 + m12 * y1 + m13, F_pt1_y = m21 * x1 + m22 * y1 + m23; const float pt2_F_x = x2 * m11 + y2 * m21 + m31, @@ -405,9 +394,6 @@ public: } return errors; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; Ptr SampsonError::create(const Mat &points) { @@ -440,9 +426,9 @@ public: m31=static_cast(m[6]); m32=static_cast(m[7]); m33=static_cast(m[8]); } - inline float getError (int point_idx) const override { - const int smpl = 4*point_idx; - const float x1=points[smpl], y1=points[smpl+1], x2=points[smpl+2], y2=points[smpl+3]; + inline float getError (int idx) const override { + idx *= 4; + const float x1=points[idx], y1=points[idx+1], x2=points[idx+2], y2=points[idx+3]; // pt2^T * E, line 1 = [l1 l2] const float l1 = x2 * m11 + y2 * m21 + m31, l2 = x2 * m12 + y2 * m22 + m32; @@ -468,9 +454,6 @@ public: } return errors; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; Ptr SymmetricGeometricDistance::create(const Mat &points) { @@ -504,10 +487,10 @@ public: p31 = (float)p[8]; p32 = (float)p[9]; p33 = (float)p[10]; p34 = (float)p[11]; } - inline float getError (int point_idx) const override { - const int smpl = 5*point_idx; - const float u = points[smpl ], v = points[smpl+1], - x = points[smpl+2], y = points[smpl+3], z = points[smpl+4]; + inline float getError (int idx) const override { + idx *= 5; + const float u = points[idx ], v = points[idx+1], + x = points[idx+2], y = points[idx+3], z = points[idx+4]; const float depth = 1 / (p31 * x + p32 * y + p33 * z + p34); const float du = u - (p11 * x + p12 * y + p13 * z + p14) * depth; const float dv = v - (p21 * x + p22 * y + p23 * z + p24) * depth; @@ -526,9 +509,6 @@ public: } return errors; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; Ptr ReprojectionErrorPmatrix::create(const Mat &points) { return makePtr(points); @@ -565,9 +545,9 @@ public: m11 = (float)m[0]; m12 = (float)m[1]; m13 = (float)m[2]; m21 = (float)m[3]; m22 = (float)m[4]; m23 = (float)m[5]; } - inline float getError (int point_idx) const override { - const int smpl = 4*point_idx; - const float x1=points[smpl], y1=points[smpl+1], x2=points[smpl+2], y2=points[smpl+3]; + inline float getError (int idx) const override { + idx *= 4; + const float x1=points[idx], y1=points[idx+1], x2=points[idx+2], y2=points[idx+3]; const float dx2 = x2 - (m11 * x1 + m12 * y1 + m13), dy2 = y2 - (m21 * x1 + m22 * y1 + m23); return dx2 * dx2 + dy2 * dy2; } @@ -581,9 +561,6 @@ public: } return errors; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; Ptr ReprojectionErrorAffine::create(const Mat &points) { @@ -606,13 +583,11 @@ public: int smpl; for (int i = 0; i < sample_size; i++) { smpl = 4 * sample[i]; - mean_pts1_x += points[smpl ]; mean_pts1_y += points[smpl + 1]; mean_pts2_x += points[smpl + 2]; mean_pts2_y += points[smpl + 3]; } - mean_pts1_x /= sample_size; mean_pts1_y /= sample_size; mean_pts2_x /= sample_size; mean_pts2_y /= sample_size; @@ -652,9 +627,9 @@ public: auto * norm_points_ptr = (float *) norm_points.data; // Normalize points: Npts = T*pts 3x3 * 3xN - const float avg_dist1f = (float)avg_dist1, avg_dist2f = (float)avg_dist2; - const float transl_x1f = (float)transl_x1, transl_y1f = (float)transl_y1; - const float transl_x2f = (float)transl_x2, transl_y2f = (float)transl_y2; + const auto avg_dist1f = (float)avg_dist1, avg_dist2f = (float)avg_dist2; + const auto transl_x1f = (float)transl_x1, transl_y1f = (float)transl_y1; + const auto transl_x2f = (float)transl_x2, transl_y2f = (float)transl_y2; for (int i = 0; i < sample_size; i++) { smpl = 4 * sample[i]; (*norm_points_ptr++) = avg_dist1f * points[smpl ] + transl_x1f; diff --git a/modules/calib3d/src/usac/fundamental_solver.cpp b/modules/calib3d/src/usac/fundamental_solver.cpp index 00d4feba63..502c3c8c62 100644 --- a/modules/calib3d/src/usac/fundamental_solver.cpp +++ b/modules/calib3d/src/usac/fundamental_solver.cpp @@ -4,19 +4,20 @@ #include "../precomp.hpp" #include "../usac.hpp" +#include "../polynom_solver.h" #ifdef HAVE_EIGEN #include #endif namespace cv { namespace usac { -// Fundamental Matrix Solver: class FundamentalMinimalSolver7ptsImpl: public FundamentalMinimalSolver7pts { private: - const Mat * points_mat; - const float * const points; + const Mat * points_mat; // pointer to OpenCV Mat + const float * const points; // pointer to points_mat->data for faster data access + const bool use_ge; public: - explicit FundamentalMinimalSolver7ptsImpl (const Mat &points_) : - points_mat (&points_), points ((float *) points_.data) {} + explicit FundamentalMinimalSolver7ptsImpl (const Mat &points_, bool use_ge_) : + points_mat (&points_), points ((float *) points_mat->data), use_ge(use_ge_) {} int estimate (const std::vector &sample, std::vector &models) const override { const int m = 7, n = 9; // rows, cols @@ -38,52 +39,57 @@ public: (*a_++) = y1; (*a_++) = 1; } - - if (!Math::eliminateUpperTriangular(a, m, n)) - return 0; - - /* - [a11 a12 a13 a14 a15 a16 a17 a18 a19] - [ 0 a22 a23 a24 a25 a26 a27 a28 a29] - [ 0 0 a33 a34 a35 a36 a37 a38 a39] - [ 0 0 0 a44 a45 a46 a47 a48 a49] - [ 0 0 0 0 a55 a56 a57 a58 a59] - [ 0 0 0 0 0 a66 a67 a68 a69] - [ 0 0 0 0 0 0 a77 a78 a79] - - f9 = 1 - */ double f1[9], f2[9]; - - f1[8] = 1.; - f1[7] = 0.; - f1[6] = -a[6*n+8] / a[6*n+6]; - - f2[8] = 1.; - f2[7] = -a[6*n+8] / a[6*n+7]; - f2[6] = 0.; - - // start from the last row - for (int i = m-2; i >= 0; i--) { - const int row_i = i*n; - double acc1 = 0, acc2 = 0; - for (int j = i+1; j < n; j++) { - acc1 -= a[row_i + j] * f1[j]; - acc2 -= a[row_i + j] * f2[j]; - } - f1[i] = acc1 / a[row_i + i]; - f2[i] = acc2 / a[row_i + i]; - - // due to numerical errors return 0 solutions - if (std::isnan(f1[i]) || std::isnan(f2[i])) + if (use_ge) { + if (!Math::eliminateUpperTriangular(a, m, n)) return 0; + /* + [a11 a12 a13 a14 a15 a16 a17 a18 a19] + [ 0 a22 a23 a24 a25 a26 a27 a28 a29] + [ 0 0 a33 a34 a35 a36 a37 a38 a39] + [ 0 0 0 a44 a45 a46 a47 a48 a49] + [ 0 0 0 0 a55 a56 a57 a58 a59] + [ 0 0 0 0 0 a66 a67 a68 a69] + [ 0 0 0 0 0 0 a77 a78 a79] + */ + + f1[8] = 1.; + f1[7] = 0.; + f1[6] = -a[6*n+8] / a[6*n+6]; + + f2[8] = 0.; + f2[7] = -a[6*n+6] / a[6*n+7]; + f2[6] = 1; + + // start from the last row + for (int i = m-2; i >= 0; i--) { + const int row_i = i*n; + double acc1 = 0, acc2 = 0; + for (int j = i+1; j < n; j++) { + acc1 -= a[row_i + j] * f1[j]; + acc2 -= a[row_i + j] * f2[j]; + } + f1[i] = acc1 / a[row_i + i]; + f2[i] = acc2 / a[row_i + i]; + + if (std::isnan(f1[i]) || std::isnan(f2[i])) + return 0; // due to numerical errors return 0 solutions + } + } else { + Mat U, Vt, D; + cv::Matx A(&a[0]); + SVD::compute(A, D, U, Vt, SVD::FULL_UV+SVD::MODIFY_A); + const auto * const vt = (double *) Vt.data; + int i1 = 8*9, i2 = 7*9; + for (int i = 0; i < 9; i++) { + f1[i] = vt[i1+i]; + f2[i] = vt[i2+i]; + } } // OpenCV: double c[4] = { 0 }, r[3] = { 0 }; double t0 = 0, t1 = 0, t2 = 0; - Mat_ coeffs (1, 4, c); - Mat_ roots (1, 3, r); for (int i = 0; i < 9; i++) f1[i] -= f2[i]; @@ -117,7 +123,7 @@ public: c[0] = f1[0]*t0 - f1[1]*t1 + f1[2]*t2; // solve the cubic equation; there can be 1 to 3 roots ... - int nroots = solveCubic (coeffs, roots); + const int nroots = solve_deg3(c[0], c[1], c[2], c[3], r[0], r[1], r[2]); if (nroots < 1) return 0; models = std::vector(nroots); @@ -145,12 +151,9 @@ public: int getMaxNumberOfSolutions () const override { return 3; } int getSampleSize() const override { return 7; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; -Ptr FundamentalMinimalSolver7pts::create(const Mat &points_) { - return makePtr(points_); +Ptr FundamentalMinimalSolver7pts::create(const Mat &points, bool use_ge) { + return makePtr(points, use_ge); } class FundamentalMinimalSolver8ptsImpl : public FundamentalMinimalSolver8pts { @@ -159,10 +162,8 @@ private: const float * const points; public: explicit FundamentalMinimalSolver8ptsImpl (const Mat &points_) : - points_mat (&points_), points ((float*) points_.data) - { - CV_DbgAssert(points); - } + points_mat (&points_), points ((float*) points_mat->data) + { CV_DbgAssert(points); } int estimate (const std::vector &sample, std::vector &models) const override { const int m = 8, n = 9; // rows, cols @@ -225,22 +226,30 @@ public: int getMaxNumberOfSolutions () const override { return 1; } int getSampleSize() const override { return 8; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; Ptr FundamentalMinimalSolver8pts::create(const Mat &points_) { return makePtr(points_); } -class FundamentalNonMinimalSolverImpl : public FundamentalNonMinimalSolver { +class EpipolarNonMinimalSolverImpl : public EpipolarNonMinimalSolver { private: const Mat * points_mat; - const Ptr normTr; + const bool do_norm; + Matx33d _T1, _T2; + Ptr normTr = nullptr; + bool enforce_rank = true, is_fundamental, use_ge; public: - explicit FundamentalNonMinimalSolverImpl (const Mat &points_) : - points_mat(&points_), normTr (NormTransform::create(points_)) {} - + explicit EpipolarNonMinimalSolverImpl (const Mat &points_, const Matx33d &T1, const Matx33d &T2, bool use_ge_) + : points_mat(&points_), do_norm(false), _T1(T1), _T2(T2), use_ge(use_ge_) { + is_fundamental = true; + } + explicit EpipolarNonMinimalSolverImpl (const Mat &points_, bool is_fundamental_) : + points_mat(&points_), do_norm(is_fundamental_), use_ge(false) { + is_fundamental = is_fundamental_; + if (is_fundamental) + normTr = NormTransform::create(points_); + } + void enforceRankConstraint (bool enforce) override { enforce_rank = enforce; } int estimate (const std::vector &sample, int sample_size, std::vector &models, const std::vector &weights) const override { if (sample_size < getMinimumRequiredSampleSize()) @@ -248,18 +257,210 @@ public: Matx33d T1, T2; Mat norm_points; - normTr->getNormTransformation(norm_points, sample, sample_size, T1, T2); - const auto * const norm_pts = (float *) norm_points.data; + if (do_norm) + normTr->getNormTransformation(norm_points, sample, sample_size, T1, T2); + const auto * const norm_pts = do_norm ? (float *) norm_points.data : (float *) points_mat->data; - // ------- 8 points algorithm with Eigen and covariance matrix -------------- + if (use_ge) { + double a[8]; + std::vector AtAb(72, 0); // 8x9 + if (weights.empty()) { + for (int i = 0; i < sample_size; i++) { + const int idx = do_norm ? 4*i : 4*sample[i]; + const double x1 = norm_pts[idx], y1 = norm_pts[idx+1], x2 = norm_pts[idx+2], y2 = norm_pts[idx+3]; + a[0] = x2*x1; + a[1] = x2*y1; + a[2] = x2; + a[3] = y2*x1; + a[4] = y2*y1; + a[5] = y2; + a[6] = x1; + a[7] = y1; + // calculate covariance for eigen + for (int row = 0; row < 8; row++) { + for (int col = row; col < 8; col++) + AtAb[row * 9 + col] += a[row] * a[col]; + AtAb[row * 9 + 8] += a[row]; + } + } + } else { // use weights + for (int i = 0; i < sample_size; i++) { + const auto weight = weights[i]; + if (weight < FLT_EPSILON) continue; + const int idx = do_norm ? 4*i : 4*sample[i]; + const double x1 = norm_pts[idx], y1 = norm_pts[idx+1], x2 = norm_pts[idx+2], y2 = norm_pts[idx+3]; + const double weight_times_x2 = weight * x2, weight_times_y2 = weight * y2; + a[0] = weight_times_x2 * x1; + a[1] = weight_times_x2 * y1; + a[2] = weight_times_x2; + a[3] = weight_times_y2 * x1; + a[4] = weight_times_y2 * y1; + a[5] = weight_times_y2; + a[6] = weight * x1; + a[7] = weight * y1; + // calculate covariance for eigen + for (int row = 0; row < 8; row++) { + for (int col = row; col < 8; col++) + AtAb[row * 9 + col] += a[row] * a[col]; + AtAb[row * 9 + 8] += a[row]; + } + } + } + // copy symmetric part of covariance matrix + for (int j = 1; j < 8; j++) + for (int z = 0; z < j; z++) + AtAb[j*9+z] = AtAb[z*9+j]; + Math::eliminateUpperTriangular(AtAb, 8, 9); + models = std::vector{ Mat_(3,3) }; + auto * f = (double *) models[0].data; + f[8] = 1.; + const int m = 8, n = 9; + // start from the last row + for (int i = m-1; i >= 0; i--) { + double acc = 0; + for (int j = i+1; j < n; j++) + acc -= AtAb[i*n+j]*f[j]; + f[i] = acc / AtAb[i*n+i]; + // due to numerical errors return 0 solutions + if (std::isnan(f[i])) + return 0; + } + } else { + // ------- 8 points algorithm with Eigen and covariance matrix -------------- + double a[9] = {0, 0, 0, 0, 0, 0, 0, 0, 1}, AtA[81] = {0}; // 9x9 + if (weights.empty()) { + for (int i = 0; i < sample_size; i++) { + const int idx = do_norm ? 4*i : 4*sample[i]; + const auto x1 = norm_pts[idx], y1 = norm_pts[idx+1], x2 = norm_pts[idx+2], y2 = norm_pts[idx+3]; + a[0] = x2*x1; + a[1] = x2*y1; + a[2] = x2; + a[3] = y2*x1; + a[4] = y2*y1; + a[5] = y2; + a[6] = x1; + a[7] = y1; + // calculate covariance matrix + for (int row = 0; row < 9; row++) + for (int col = row; col < 9; col++) + AtA[row*9+col] += a[row]*a[col]; + } + } else { // use weights + for (int i = 0; i < sample_size; i++) { + const auto weight = weights[i]; + if (weight < FLT_EPSILON) continue; + const int smpl = do_norm ? 4*i : 4*sample[i]; + const auto x1 = norm_pts[smpl], y1 = norm_pts[smpl+1], x2 = norm_pts[smpl+2], y2 = norm_pts[smpl+3]; + const double weight_times_x2 = weight * x2, weight_times_y2 = weight * y2; + a[0] = weight_times_x2 * x1; + a[1] = weight_times_x2 * y1; + a[2] = weight_times_x2; + a[3] = weight_times_y2 * x1; + a[4] = weight_times_y2 * y1; + a[5] = weight_times_y2; + a[6] = weight * x1; + a[7] = weight * y1; + a[8] = weight; + for (int row = 0; row < 9; row++) + for (int col = row; col < 9; col++) + AtA[row*9+col] += a[row]*a[col]; + } + } + for (int j = 1; j < 9; j++) + for (int z = 0; z < j; z++) + AtA[j*9+z] = AtA[z*9+j]; +#ifdef HAVE_EIGEN + models = std::vector{ Mat_(3,3) }; + // extract the last null-vector + Eigen::Map>((double *)models[0].data) = Eigen::JacobiSVD + > ((Eigen::Matrix(AtA)), + Eigen::ComputeFullV).matrixV().col(8); +#else + Matx AtA_(AtA), U, Vt; + Vec W; + SVD::compute(AtA_, W, U, Vt, SVD::FULL_UV + SVD::MODIFY_A); + models = std::vector { Mat_(3, 3, Vt.val + 72 /*=8*9*/) }; +#endif + } + + if (enforce_rank) + FundamentalDegeneracy::recoverRank(models[0], is_fundamental); + if (is_fundamental) { + const auto * const f = (double *) models[0].data; + const auto * const t1 = do_norm ? T1.val : _T1.val, * t2 = do_norm ? T2.val : _T2.val; + // F = T2^T F T1 + models[0] = Mat(Matx33d(t1[0]*t2[0]*f[0],t1[0]*t2[0]*f[1], t2[0]*f[2] + t2[0]*f[0]*t1[2] + + t2[0]*f[1]*t1[5], t1[0]*t2[0]*f[3],t1[0]*t2[0]*f[4], t2[0]*f[5] + t2[0]*f[3]*t1[2] + + t2[0]*f[4]*t1[5], t1[0]*(f[6] + f[0]*t2[2] + f[3]*t2[5]), t1[0]*(f[7] + f[1]*t2[2] + + f[4]*t2[5]), f[8] + t1[2]*(f[6] + f[0]*t2[2] + f[3]*t2[5]) + t1[5]*(f[7] + f[1]*t2[2] + + f[4]*t2[5]) + f[2]*t2[2] + f[5]*t2[5])); + } + return 1; + } + int estimate (const std::vector &/*mask*/, std::vector &/*models*/, + const std::vector &/*weights*/) override { + return 0; + } + int getMinimumRequiredSampleSize() const override { return 8; } + int getMaxNumberOfSolutions () const override { return 1; } +}; +Ptr EpipolarNonMinimalSolver::create(const Mat &points_, bool is_fundamental) { + return makePtr(points_, is_fundamental); +} +Ptr EpipolarNonMinimalSolver::create(const Mat &points_, const Matx33d &T1, const Matx33d &T2, bool use_ge) { + return makePtr(points_, T1, T2, use_ge); +} + +class CovarianceEpipolarSolverImpl : public CovarianceEpipolarSolver { +private: + Mat norm_pts; + Matx33d T1, T2; + float * norm_points; + std::vector mask; + int points_size; + double covariance[81] = {0}, * t1, * t2; + bool is_fundamental, enforce_rank = true; +public: + explicit CovarianceEpipolarSolverImpl (const Mat &norm_points_, const Matx33d &T1_, const Matx33d &T2_) + : norm_pts(norm_points_), T1(T1_), T2(T2_) { + points_size = norm_points_.rows; + norm_points = (float *) norm_pts.data; + t1 = T1.val; t2 = T2.val; + mask = std::vector(points_size, false); + is_fundamental = true; + } + explicit CovarianceEpipolarSolverImpl (const Mat &points_, bool is_fundamental_) { + points_size = points_.rows; + is_fundamental = is_fundamental_; + if (is_fundamental) { // normalize image points only for fundmantal matrix + std::vector sample(points_size); + for (int i = 0; i < points_size; i++) sample[i] = i; + const Ptr normTr = NormTransform::create(points_); + normTr->getNormTransformation(norm_pts, sample, points_size, T1, T2); + t1 = T1.val; t2 = T2.val; + } else norm_pts = points_; // otherwise points are normalized by intrinsics + norm_points = (float *)norm_pts.data; + mask = std::vector(points_size, false); + } + void enforceRankConstraint (bool enforce_) override { enforce_rank = enforce_; } + + void reset () override { + std::fill(covariance, covariance+81, 0); + std::fill(mask.begin(), mask.end(), false); + } + /* + * Find fundamental matrix using 8-point algorithm with covariance matrix and PCA + */ + int estimate (const std::vector &new_mask, std::vector &models, + const std::vector &/*weights*/) override { double a[9] = {0, 0, 0, 0, 0, 0, 0, 0, 1}; - double AtA[81] = {0}; // 9x9 - if (weights.empty()) { - for (int i = 0; i < sample_size; i++) { - const int norm_points_idx = 4*i; - const double x1 = norm_pts[norm_points_idx ], y1 = norm_pts[norm_points_idx+1], - x2 = norm_pts[norm_points_idx+2], y2 = norm_pts[norm_points_idx+3]; + for (int i = 0; i < points_size; i++) { + if (mask[i] != new_mask[i]) { + const int smpl = 4*i; + const double x1 = norm_points[smpl ], y1 = norm_points[smpl+1], + x2 = norm_points[smpl+2], y2 = norm_points[smpl+3]; + a[0] = x2*x1; a[1] = x2*y1; a[2] = x2; @@ -269,71 +470,129 @@ public: a[6] = x1; a[7] = y1; - // calculate covariance for eigen - for (int row = 0; row < 9; row++) - for (int col = row; col < 9; col++) - AtA[row*9+col] += a[row]*a[col]; - } - } else { - for (int i = 0; i < sample_size; i++) { - const int smpl = 4*i; - const double weight = weights[i]; - const double x1 = norm_pts[smpl ], y1 = norm_pts[smpl+1], - x2 = norm_pts[smpl+2], y2 = norm_pts[smpl+3]; - const double weight_times_x2 = weight * x2, - weight_times_y2 = weight * y2; - - a[0] = weight_times_x2 * x1; - a[1] = weight_times_x2 * y1; - a[2] = weight_times_x2; - a[3] = weight_times_y2 * x1; - a[4] = weight_times_y2 * y1; - a[5] = weight_times_y2; - a[6] = weight * x1; - a[7] = weight * y1; - a[8] = weight; - - // calculate covariance for eigen - for (int row = 0; row < 9; row++) - for (int col = row; col < 9; col++) - AtA[row*9+col] += a[row]*a[col]; + if (mask[i]) // if mask[i] is true then new_mask[i] must be false + for (int j = 0; j < 9; j++) + for (int z = j; z < 9; z++) + covariance[j*9+z] -= a[j]*a[z]; + else + for (int j = 0; j < 9; j++) + for (int z = j; z < 9; z++) + covariance[j*9+z] += a[j]*a[z]; } } + mask = new_mask; // copy symmetric part of covariance matrix for (int j = 1; j < 9; j++) for (int z = 0; z < j; z++) - AtA[j*9+z] = AtA[z*9+j]; + covariance[j*9+z] = covariance[z*9+j]; #ifdef HAVE_EIGEN models = std::vector{ Mat_(3,3) }; - const Eigen::JacobiSVD> svd((Eigen::Matrix(AtA)), - Eigen::ComputeFullV); - // extract the last nullspace - Eigen::Map>((double *)models[0].data) = svd.matrixV().col(8); + // extract the last null-vector + Eigen::Map>((double *)models[0].data) = Eigen::JacobiSVD + > ((Eigen::Matrix(covariance)), + Eigen::ComputeFullV).matrixV().col(8); #else - Matx AtA_(AtA), U, Vt; - Vec W; - SVD::compute(AtA_, W, U, Vt, SVD::FULL_UV + SVD::MODIFY_A); - models = std::vector { Mat_(3, 3, Vt.val + 72 /*=8*9*/) }; + Matx AtA_(covariance), U, Vt; + Vec W; + SVD::compute(AtA_, W, U, Vt, SVD::FULL_UV + SVD::MODIFY_A); + models = std::vector { Mat_(3, 3, Vt.val + 72 /*=8*9*/) }; #endif - FundamentalDegeneracy::recoverRank(models[0], true/*F*/); + if (enforce_rank) + FundamentalDegeneracy::recoverRank(models[0], is_fundamental); + if (is_fundamental) { + const auto * const f = (double *) models[0].data; + // F = T2^T F T1 + models[0] = Mat(Matx33d(t1[0]*t2[0]*f[0],t1[0]*t2[0]*f[1], t2[0]*f[2] + t2[0]*f[0]*t1[2] + + t2[0]*f[1]*t1[5], t1[0]*t2[0]*f[3],t1[0]*t2[0]*f[4], t2[0]*f[5] + t2[0]*f[3]*t1[2] + + t2[0]*f[4]*t1[5], t1[0]*(f[6] + f[0]*t2[2] + f[3]*t2[5]), t1[0]*(f[7] + f[1]*t2[2] + + f[4]*t2[5]), f[8] + t1[2]*(f[6] + f[0]*t2[2] + f[3]*t2[5]) + t1[5]*(f[7] + f[1]*t2[2] + + f[4]*t2[5]) + f[2]*t2[2] + f[5]*t2[5])); + } + return 1; + } + int getMinimumRequiredSampleSize() const override { return 8; } + int getMaxNumberOfSolutions () const override { return 1; } +}; +Ptr CovarianceEpipolarSolver::create (const Mat &points, bool is_fundamental) { + return makePtr(points, is_fundamental); +} +Ptr CovarianceEpipolarSolver::create (const Mat &points, const Matx33d &T1, const Matx33d &T2) { + return makePtr(points, T1, T2); +} - // Transpose T2 (in T2 the lower diagonal is zero) - T2(2, 0) = T2(0, 2); T2(2, 1) = T2(1, 2); - T2(0, 2) = 0; T2(1, 2) = 0; +class LarssonOptimizerImpl : public LarssonOptimizer { +private: + const Mat &calib_points; + Matx33d K1, K2, K2_t, K1_inv, K2_inv_t; + bool is_fundamental; + BundleOptions opt; +public: + LarssonOptimizerImpl (const Mat &calib_points_, const Matx33d &K1_, const Matx33d &K2_, int max_iters_, bool is_fundamental_) : + calib_points(calib_points_), K1(K1_), K2(K2_){ + is_fundamental = is_fundamental_; + opt.max_iterations = max_iters_; + opt.loss_scale = Utils::getCalibratedThreshold(std::max(1.5, opt.loss_scale), Mat(K1), Mat(K2)); + if (is_fundamental) { + K1_inv = K1.inv(); + K2_t = K2.t(); + K2_inv_t = K2_t.inv(); + } + } - models[0] = T2 * models[0] * T1; + int estimate (const Mat &model, const std::vector &sample, int sample_size, std::vector + &models, const std::vector &weights) const override { + if (sample_size < 5) return 0; + const Matx33d E = is_fundamental ? K2_t * Matx33d(model) * K1 : model; + RNG rng (sample_size); + cv::Matx33d R1, R2; cv::Vec3d t; + cv::decomposeEssentialMat(E, R1, R2, t); + int positive_depth[4] = {0}; + const auto * const pts_ = (float *) calib_points.data; + // a few point are enough to test + // actually due to Sampson error minimization, the input R,t do not really matter + // for a correct pair there is a sligthly faster convergence + for (int i = 0; i < 3; i++) { // could be 1 point + const int rand_inl = 4 * sample[rng.uniform(0, sample_size)]; + Vec3d p1 (pts_[rand_inl], pts_[rand_inl+1], 1), p2(pts_[rand_inl+2], pts_[rand_inl+3], 1); + p1 /= norm(p1); p2 /= norm(p2); + if (satisfyCheirality(R1, t, p1, p2)) positive_depth[0]++; + if (satisfyCheirality(R1, -t, p1, p2)) positive_depth[1]++; + if (satisfyCheirality(R2, t, p1, p2)) positive_depth[2]++; + if (satisfyCheirality(R2, -t, p1, p2)) positive_depth[3]++; + } + int corr_idx = 0, max_good_pts = positive_depth[0]; + for (int i = 1; i < 4; i++) { + if (max_good_pts < positive_depth[i]) { + max_good_pts = positive_depth[i]; + corr_idx = i; + } + } + + CameraPose pose; + pose.R = corr_idx < 2 ? R1 : R2; + pose.t = corr_idx % 2 == 1 ? -t : t; + refine_relpose(calib_points, sample, sample_size, &pose, opt, &weights[0]); + Matx33d model_new = Math::getSkewSymmetric(pose.t) * pose.R; + if (is_fundamental) + model_new = K2_inv_t * model_new * K1_inv; + models = std::vector { Mat(model_new) }; return 1; } - int getMinimumRequiredSampleSize() const override { return 8; } - int getMaxNumberOfSolutions () const override { return 1; } - Ptr clone () const override { - return makePtr(*points_mat); + int estimate (const std::vector&, int, std::vector&, const std::vector&) const override { + return 0; } + int estimate (const std::vector &/*mask*/, std::vector &/*models*/, + const std::vector &/*weights*/) override { + return 0; + } + void enforceRankConstraint (bool /*enforce*/) override {} + int getMinimumRequiredSampleSize() const override { return 5; } + int getMaxNumberOfSolutions () const override { return 1; } }; -Ptr FundamentalNonMinimalSolver::create(const Mat &points_) { - return makePtr(points_); +Ptr LarssonOptimizer::create(const Mat &calib_points_, const Matx33d &K1, const Matx33d &K2, int max_iters_, bool is_fundamental) { + return makePtr(calib_points_, K1, K2, max_iters_, is_fundamental); } }} diff --git a/modules/calib3d/src/usac/gamma_values.cpp b/modules/calib3d/src/usac/gamma_values.cpp index 1e82d8eba7..fa1c03e8ed 100644 --- a/modules/calib3d/src/usac/gamma_values.cpp +++ b/modules/calib3d/src/usac/gamma_values.cpp @@ -7,101 +7,146 @@ namespace cv { namespace usac { -GammaValues::GammaValues() - : max_range_complete(4.62) - , max_range_gamma(1.52) - , max_size_table(3000) -{ - /* - * Gamma values for degrees of freedom n = 2 and sigma quantile 99% of chi distribution - * (squared root of chi-squared distribution), in the range <0; 4.62> for complete values - * and <0, 1.52> for gamma values. - * Number of anchor points is 50. Other values are approximated using linear interpolation - */ - const int number_of_anchor_points = 50; - std::vector gamma_complete_anchor = std::vector - {1.7724538509055159, 1.182606138403832, 0.962685372890749, 0.8090013493715409, - 0.6909325812483967, 0.5961199186942078, 0.5179833984918483, 0.45248091153099873, - 0.39690029823142897, 0.34930995878395804, 0.3082742109224103, 0.2726914551904204, - 0.2416954924567404, 0.21459196516027726, 0.190815580770884, 0.16990026519723456, - 0.15145770273372564, 0.13516150988807635, 0.12073530906427948, 0.10794357255251595, - 0.0965844793065712, 0.08648426334883624, 0.07749268706639856, 0.06947937608738222, - 0.062330823249820304, 0.05594791865006951, 0.05024389794830681, 0.045142626552664405, - 0.040577155977706246, 0.03648850256745103, 0.03282460924226794, 0.029539458909083157, - 0.02659231432268328, 0.023947063970062663, 0.021571657306774475, 0.01943761564987864, - 0.017519607407598645, 0.015795078236273064, 0.014243928262247118, 0.012848229767187478, - 0.011591979769030827, 0.010460882783057988, 0.009442159753944173, 0.008524379737926344, - 0.007697311406424555, 0.006951791856026042, 0.006279610558635573, 0.005673406581042374, - 0.005126577454218803, 0.004633198286725555}; +class GammaValuesImpl : public GammaValues { + std::vector gamma_complete, gamma_incomplete, gamma; + double scale_complete_values, scale_gamma_values; + int max_size_table, DoF; +public: + GammaValuesImpl (int DoF_, int max_size_table_) { + max_size_table = max_size_table_; + max_size_table = max_size_table_; + DoF = DoF_; + /* + * Gamma values for degrees of freedom n = 2 and sigma quantile 99% of chi distribution + * (squared root of chi-squared distribution), in the range <0; 4.62> for complete values + * and <0, 1.52> for gamma values. + * Number of anchor points is 50. Other values are approximated using linear interpolation + */ + const int number_of_anchor_points = 50; + std::vector gamma_complete_anchor, gamma_incomplete_anchor, gamma_anchor; + if (DoF == 2) { + const double max_thr = 7.5, gamma_quantile = 3.04; + scale_complete_values = max_size_table_ / max_thr; + scale_gamma_values = gamma_quantile * max_size_table_ / max_thr ; - std::vector gamma_incomplete_anchor = std::vector - {0.0, 0.01773096912803939, 0.047486924846289004, 0.08265437835139826, 0.120639343491371, - 0.15993024714868515, 0.19954558593754865, 0.23881753504915218, 0.2772830648361923, - 0.3146208784488923, 0.3506114446939783, 0.385110056889967, 0.41802785670077697, - 0.44931803198258047, 0.47896553567848993, 0.5069792897777948, 0.5333861945970247, - 0.5582264802664578, 0.581550074874317, 0.6034137543595729, 0.6238789008764282, - 0.6430097394182639, 0.6608719532994989, 0.6775316015953519, 0.6930542783709592, - 0.7075044661695132, 0.7209450459078338, 0.733436932830201, 0.7450388140484766, - 0.7558069678435577, 0.7657951486073097, 0.7750545242776943, 0.7836336555215403, - 0.7915785078697124, 0.798932489600361, 0.8057365094688473, 0.8120290494534339, - 0.8178462485678104, 0.8232219945197348, 0.8281880205973585, 0.8327740056635289, - 0.8370076755516281, 0.8409149044990385, 0.8445198155381767, 0.8478448790000731, - 0.8509110084798414, 0.8537376537738418, 0.8563428904304485, 0.8587435056647642, - 0.8609550804762539}; + gamma_complete_anchor = std::vector + {1.77245385e+00, 1.02824699e+00, 7.69267629e-01, 5.99047749e-01, + 4.75998050e-01, 3.83008633e-01, 3.10886473e-01, 2.53983661e-01, + 2.08540472e-01, 1.71918718e-01, 1.42197872e-01, 1.17941854e-01, + 9.80549104e-02, 8.16877552e-02, 6.81739145e-02, 5.69851046e-02, + 4.76991202e-02, 3.99417329e-02, 3.35126632e-02, 2.81470710e-02, + 2.36624697e-02, 1.99092598e-02, 1.67644090e-02, 1.41264487e-02, + 1.19114860e-02, 1.00500046e-02, 8.48428689e-03, 7.16632498e-03, + 6.05612291e-03, 5.12031042e-03, 4.33100803e-03, 3.66489504e-03, + 3.10244213e-03, 2.62514027e-03, 2.22385863e-03, 1.88454040e-03, + 1.59749690e-03, 1.35457835e-03, 1.14892453e-03, 9.74756909e-04, + 8.27205063e-04, 7.02161552e-04, 5.96160506e-04, 5.06275903e-04, + 4.30036278e-04, 3.65353149e-04, 3.10460901e-04, 2.63866261e-04, + 2.24305797e-04, 1.90558599e-04}; - std::vector gamma_anchor = std::vector - {1.7724538509055159, 1.427187162582056, 1.2890382454046982, 1.186244737282388, - 1.1021938955410173, 1.0303674512016956, 0.9673796229113404, 0.9111932804012203, - 0.8604640514722175, 0.814246149432561, 0.7718421763436497, 0.7327190195355812, - 0.6964573670982434, 0.6627197089339725, 0.6312291454822467, 0.6017548373556638, - 0.5741017071093776, 0.5481029597580317, 0.523614528104858, 0.5005108666212138, - 0.478681711577816, 0.4580295473431646, 0.43846759792922513, 0.41991821541471996, - 0.40231157253054745, 0.38558459136185, 0.3696800574963841, 0.3545458813847714, - 0.340134477710645, 0.32640224021796493, 0.3133090943985706, 0.3008181141790485, - 0.28889519159238314, 0.2775087506098113, 0.2666294980086962, 0.2562302054837794, - 0.24628551826026082, 0.2367717863030556, 0.22766691488600885, 0.21895023182476064, - 0.2106023691144937, 0.2026051570714723, 0.19494152937027823, 0.18759543761063277, - 0.1805517742482484, 0.17379630289125447, 0.16731559510356395, 0.1610969729740903, - 0.1551284568099053, 0.14939871739550692}; + gamma_incomplete_anchor = std::vector + {0. , 0.0364325 , 0.09423626, 0.15858163, 0.22401622, 0.28773243, + 0.34820493, 0.40463362, 0.45665762, 0.50419009, 0.54731575, 0.58622491, + 0.62116968, 0.65243473, 0.68031763, 0.70511575, 0.72711773, 0.74668782, + 0.76389332, 0.77907386, 0.79244816, 0.80421561, 0.81455692, 0.8236351 , + 0.83159653, 0.83857228, 0.84467927, 0.85002158, 0.85469163, 0.85877132, + 0.86233307, 0.86544086, 0.86815108, 0.87052421, 0.87258093, 0.87437198, + 0.87593103, 0.87728759, 0.87846752, 0.87949345, 0.88038518, 0.88116002, + 0.88183307, 0.88241755, 0.88292497, 0.88336537, 0.88374751, 0.88407901, + 0.88436652, 0.88461696}; - // allocate tables - gamma_complete = std::vector(max_size_table); - gamma_incomplete = std::vector(max_size_table); - gamma = std::vector(max_size_table); + gamma_anchor = std::vector + {1.77245385e+00, 5.93375722e-01, 3.05833272e-01, 1.68019955e-01, + 9.52188705e-02, 5.49876141e-02, 3.21629603e-02, 1.89881161e-02, + 1.12897384e-02, 6.75016002e-03, 4.05426969e-03, 2.44422283e-03, + 1.47822780e-03, 8.96425642e-04, 5.44879754e-04, 3.31873268e-04, + 2.02499478e-04, 1.23458651e-04, 7.55593392e-05, 4.63032752e-05, + 2.84078946e-05, 1.74471428e-05, 1.07257506e-05, 6.59955061e-06, + 4.06400013e-06, 2.50448635e-06, 1.54449028e-06, 9.53085308e-07, + 5.88490160e-07, 3.63571768e-07, 2.24734099e-07, 1.38982938e-07, + 8.59913580e-08, 5.31026827e-08, 3.28834964e-08, 2.03707922e-08, + 1.26240063e-08, 7.82595771e-09, 4.85312084e-09, 3.01051575e-09, + 1.86805770e-09, 1.15947962e-09, 7.19869372e-10, 4.47050615e-10, + 2.77694421e-10, 1.72536278e-10, 1.07224039e-10, 6.66497131e-11, + 4.14376355e-11, 2.57079508e-11}; + } else if (DoF == 4) { + const double max_thr = 2.5, gamma_quantile = 3.64; + scale_complete_values = max_size_table_ / max_thr; + scale_gamma_values = gamma_quantile * max_size_table_ / max_thr ; + gamma_complete_anchor = std::vector + {0.88622693, 0.87877828, 0.86578847, 0.84979442, 0.83179176, 0.81238452, + 0.79199067, 0.77091934, 0.74940836, 0.72764529, 0.70578051, 0.68393585, + 0.66221071, 0.64068639, 0.61942952, 0.59849449, 0.57792561, 0.55766078, + 0.53792634, 0.51864482, 0.49983336, 0.48150466, 0.46366759, 0.44632776, + 0.42948797, 0.41314862, 0.39730804, 0.38196282, 0.36710806, 0.35273761, + 0.33884422, 0.32541979, 0.31245545, 0.29988151, 0.28781065, 0.2761701 , + 0.26494924, 0.25413723, 0.24372308, 0.23369573, 0.22404405, 0.21475696, + 0.2058234 , 0.19723241, 0.18897314, 0.18103488, 0.17340708, 0.16607937, + 0.15904157, 0.15225125}; + gamma_incomplete_anchor = std::vector + {0.00000000e+00, 2.26619558e-04, 1.23631005e-03, 3.28596265e-03, + 6.50682297e-03, 1.09662062e-02, 1.66907233e-02, 2.36788942e-02, + 3.19091043e-02, 4.13450655e-02, 5.19397673e-02, 6.36384378e-02, + 7.63808171e-02, 9.01029320e-02, 1.04738496e-01, 1.20220023e-01, + 1.36479717e-01, 1.53535010e-01, 1.71152805e-01, 1.89349599e-01, + 2.08062142e-01, 2.27229225e-01, 2.46791879e-01, 2.66693534e-01, + 2.86880123e-01, 3.07300152e-01, 3.27904735e-01, 3.48647611e-01, + 3.69485130e-01, 3.90376227e-01, 4.11282379e-01, 4.32167556e-01, + 4.52998149e-01, 4.73844336e-01, 4.94473655e-01, 5.14961263e-01, + 5.35282509e-01, 5.55414767e-01, 5.75337352e-01, 5.95031429e-01, + 6.14479929e-01, 6.33667460e-01, 6.52580220e-01, 6.71205917e-01, + 6.89533681e-01, 7.07553988e-01, 7.25258581e-01, 7.42640393e-01, + 7.59693477e-01, 7.76494059e-01}; + gamma_anchor = std::vector + {8.86226925e-01, 8.38460922e-01, 7.64931722e-01, 6.85680218e-01, + 6.07663201e-01, 5.34128389e-01, 4.66574835e-01, 4.05560768e-01, + 3.51114357e-01, 3.02965249e-01, 2.60682396e-01, 2.23758335e-01, + 1.91661077e-01, 1.63865725e-01, 1.39873108e-01, 1.19220033e-01, + 1.01484113e-01, 8.62162923e-02, 7.32253576e-02, 6.21314285e-02, + 5.26713657e-02, 4.46151697e-02, 3.77626859e-02, 3.19403783e-02, + 2.69982683e-02, 2.28070945e-02, 1.92557199e-02, 1.62487939e-02, + 1.37046640e-02, 1.15535264e-02, 9.73579631e-03, 8.20068208e-03, + 6.90494092e-03, 5.80688564e-03, 4.88587254e-03, 4.10958296e-03, + 3.45555079e-03, 2.90474053e-03, 2.44103551e-03, 2.05079975e-03, + 1.72250366e-03, 1.44640449e-03, 1.21427410e-03, 1.01916714e-03, + 8.55224023e-04, 7.17503448e-04, 6.01840372e-04, 5.04725511e-04, + 4.23203257e-04, 3.54478559e-04}; + } else CV_Error(cv::Error::StsNotImplemented, "Not implemented for specific DoF!"); + // allocate tables + gamma_complete = std::vector(max_size_table); + gamma_incomplete = std::vector(max_size_table); + gamma = std::vector(max_size_table); - const int step = (int)((double)max_size_table / (number_of_anchor_points-1)); - int arr_cnt = 0; - for (int i = 0; i < number_of_anchor_points-1; i++) { - const double complete_x0 = gamma_complete_anchor[i], step_complete = (gamma_complete_anchor[i+1] - complete_x0) / step; - const double incomplete_x0 = gamma_incomplete_anchor[i], step_incomplete = (gamma_incomplete_anchor[i+1] - incomplete_x0) / step; - const double gamma_x0 = gamma_anchor[i], step_gamma = (gamma_anchor[i+1] - gamma_x0) / step; + // do linear interpolation of gamma values + const int step = (int)((double)max_size_table / (number_of_anchor_points-1)); + int arr_cnt = 0; + for (int i = 0; i < number_of_anchor_points-1; i++) { + const double complete_x0 = gamma_complete_anchor[i], step_complete = (gamma_complete_anchor[i+1] - complete_x0) / step; + const double incomplete_x0 = gamma_incomplete_anchor[i], step_incomplete = (gamma_incomplete_anchor[i+1] - incomplete_x0) / step; + const double gamma_x0 = gamma_anchor[i], step_gamma = (gamma_anchor[i+1] - gamma_x0) / step; - for (int j = 0; j < step; j++) { - gamma_complete[arr_cnt] = complete_x0 + j * step_complete; - gamma_incomplete[arr_cnt] = incomplete_x0 + j * step_incomplete; - gamma[arr_cnt++] = gamma_x0 + j * step_gamma; - } - } - if (arr_cnt < max_size_table) { - // if array was not totally filled (in some cases can happen) then copy last values - std::fill(gamma_complete.begin()+arr_cnt, gamma_complete.end(), gamma_complete[arr_cnt-1]); - std::fill(gamma_incomplete.begin()+arr_cnt, gamma_incomplete.end(), gamma_incomplete[arr_cnt-1]); - std::fill(gamma.begin()+arr_cnt, gamma.end(), gamma[arr_cnt-1]); + for (int j = 0; j < step; j++) { + gamma_complete[arr_cnt] = complete_x0 + j * step_complete; + gamma_incomplete[arr_cnt] = incomplete_x0 + j * step_incomplete; + gamma[arr_cnt++] = gamma_x0 + j * step_gamma; + } + } + if (arr_cnt < max_size_table) { + // if array was not totally filled (in some cases can happen) then copy last values + std::fill(gamma_complete.begin()+arr_cnt, gamma_complete.end(), gamma_complete[arr_cnt-1]); + std::fill(gamma_incomplete.begin()+arr_cnt, gamma_incomplete.end(), gamma_incomplete[arr_cnt-1]); + std::fill(gamma.begin()+arr_cnt, gamma.end(), gamma[arr_cnt-1]); + } } + + const std::vector &getCompleteGammaValues() const override { return gamma_complete; } + const std::vector &getIncompleteGammaValues() const override { return gamma_incomplete; } + const std::vector &getGammaValues() const override { return gamma; } + double getScaleOfGammaCompleteValues () const override { return scale_complete_values; } + double getScaleOfGammaValues () const override { return scale_gamma_values; } + int getTableSize () const override { return max_size_table; } +}; +Ptr GammaValues::create(int DoF, int max_size_table) { + return makePtr(DoF, max_size_table); } - -const std::vector& GammaValues::getCompleteGammaValues() const { return gamma_complete; } -const std::vector& GammaValues::getIncompleteGammaValues() const { return gamma_incomplete; } -const std::vector& GammaValues::getGammaValues() const { return gamma; } -double GammaValues::getScaleOfGammaCompleteValues () const { return gamma_complete.size() / max_range_complete; } -double GammaValues::getScaleOfGammaValues () const { return gamma.size() / max_range_gamma; } -int GammaValues::getTableSize () const { return max_size_table; } - -/* static */ -const GammaValues& GammaValues::getSingleton() -{ - static GammaValues g_gammaValues; - return g_gammaValues; -} - }} // namespace diff --git a/modules/calib3d/src/usac/homography_solver.cpp b/modules/calib3d/src/usac/homography_solver.cpp index b60e39781a..ba5e233d22 100644 --- a/modules/calib3d/src/usac/homography_solver.cpp +++ b/modules/calib3d/src/usac/homography_solver.cpp @@ -9,13 +9,14 @@ #endif namespace cv { namespace usac { -class HomographyMinimalSolver4ptsGEMImpl : public HomographyMinimalSolver4ptsGEM { +class HomographyMinimalSolver4ptsImpl : public HomographyMinimalSolver4pts { private: const Mat * points_mat; const float * const points; + const bool use_ge; public: - explicit HomographyMinimalSolver4ptsGEMImpl (const Mat &points_) : - points_mat(&points_), points ((float*) points_.data) {} + explicit HomographyMinimalSolver4ptsImpl (const Mat &points_, bool use_ge_) : + points_mat(&points_), points ((float*) points_mat->data), use_ge(use_ge_) {} int estimate (const std::vector& sample, std::vector &models) const override { int m = 8, n = 9; @@ -23,7 +24,7 @@ public: int cnt = 0; for (int i = 0; i < 4; i++) { const int smpl = 4*sample[i]; - const double x1 = points[smpl], y1 = points[smpl+1], x2 = points[smpl+2], y2 = points[smpl+3]; + const auto x1 = points[smpl], y1 = points[smpl+1], x2 = points[smpl+2], y2 = points[smpl+3]; A[cnt++] = -x1; A[cnt++] = -y1; @@ -42,49 +43,53 @@ public: A[cnt++] = y2; } - if (!Math::eliminateUpperTriangular(A, m, n)) - return 0; - - models = std::vector{ Mat_(3,3) }; - auto * h = (double *) models[0].data; - h[8] = 1.; - - // start from the last row - for (int i = m-1; i >= 0; i--) { - double acc = 0; - for (int j = i+1; j < n; j++) - acc -= A[i*n+j]*h[j]; - - h[i] = acc / A[i*n+i]; - // due to numerical errors return 0 solutions - if (std::isnan(h[i])) + if (use_ge) { + if (!Math::eliminateUpperTriangular(A, m, n)) return 0; + + models = std::vector{ Mat_(3,3) }; + auto * h = (double *) models[0].data; + h[8] = 1.; + + // start from the last row + for (int i = m-1; i >= 0; i--) { + double acc = 0; + for (int j = i+1; j < n; j++) + acc -= A[i*n+j]*h[j]; + + h[i] = acc / A[i*n+i]; + // due to numerical errors return 0 solutions + if (std::isnan(h[i])) + return 0; + } + } else { + Mat U, Vt, D; + cv::Matx A_svd(&A[0]); + SVD::compute(A_svd, D, U, Vt, SVD::FULL_UV+SVD::MODIFY_A); + models = std::vector { Vt.row(Vt.rows-1).reshape(0, 3) }; } return 1; } int getMaxNumberOfSolutions () const override { return 1; } int getSampleSize() const override { return 4; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; -Ptr HomographyMinimalSolver4ptsGEM::create(const Mat &points_) { - return makePtr(points_); +Ptr HomographyMinimalSolver4pts::create(const Mat &points, bool use_ge) { + return makePtr(points, use_ge); } class HomographyNonMinimalSolverImpl : public HomographyNonMinimalSolver { private: const Mat * points_mat; - const Ptr normTr; + const bool do_norm, use_ge; + Ptr normTr; + Matx33d _T1, _T2; public: - explicit HomographyNonMinimalSolverImpl (const Mat &points_) : - points_mat(&points_), normTr (NormTransform::create(points_)) {} + explicit HomographyNonMinimalSolverImpl (const Mat &norm_points_, const Matx33d &T1, const Matx33d &T2, bool use_ge_) : + points_mat(&norm_points_), do_norm(false), use_ge(use_ge_), _T1(T1), _T2(T2) {} + explicit HomographyNonMinimalSolverImpl (const Mat &points_, bool use_ge_) : + points_mat(&points_), do_norm(true), use_ge(use_ge_), normTr (NormTransform::create(points_)) {} - /* - * Find Homography matrix using (weighted) non-minimal estimation. - * Use Principal Component Analysis. Use normalized points. - */ int estimate (const std::vector &sample, int sample_size, std::vector &models, const std::vector &weights) const override { if (sample_size < getMinimumRequiredSampleSize()) @@ -92,21 +97,226 @@ public: Matx33d T1, T2; Mat norm_points_; - normTr->getNormTransformation(norm_points_, sample, sample_size, T1, T2); + if (do_norm) + normTr->getNormTransformation(norm_points_, sample, sample_size, T1, T2); + const auto * const npts = do_norm ? (float *) norm_points_.data : (float *) points_mat->data; - /* - * @norm_points is matrix 4 x inlier_size - * @weights is vector of inliers_size - * weights[i] is weight of i-th inlier - */ - const auto * const norm_points = (float *) norm_points_.data; + Mat H; + if (use_ge) { + double a1[8] = {0, 0, -1, 0, 0, 0, 0, 0}, + a2[8] = {0, 0, 0, 0, 0, -1, 0, 0}; + std::vector AtAb(72, 0); // 8x9 + if (weights.empty()) { + for (int i = 0; i < sample_size; i++) { + const int idx = do_norm ? 4*i : 4*sample[i]; + const double x1 = npts[idx], y1 = npts[idx+1], x2 = npts[idx+2], y2 = npts[idx+3]; + a1[0] = -x1; + a1[1] = -y1; + a1[6] = x2*x1; + a1[7] = x2*y1; + a2[3] = -x1; + a2[4] = -y1; + a2[6] = y2*x1; + a2[7] = y2*y1; + + // calculate covariance for eigen + for (int j = 0; j < 8; j++) { + for (int z = j; z < 8; z++) + AtAb[j * 9 + z] += a1[j]*a1[z] + a2[j]*a2[z]; + AtAb[j * 9 + 8] += a1[j]*x2 + a2[j]*y2; + } + } + } else { // use weights + for (int i = 0; i < sample_size; i++) { + const double weight = weights[i]; + if (weight < FLT_EPSILON) continue; + const int idx = do_norm ? 4*i : 4*sample[i]; + const double x1 = npts[idx], y1 = npts[idx+1], x2 = npts[idx+2], y2 = npts[idx+3]; + const double minus_weight_times_x1 = -weight * x1, + minus_weight_times_y1 = -weight * y1, + weight_times_x2 = weight * x2, + weight_times_y2 = weight * y2; + + a1[0] = minus_weight_times_x1; + a1[1] = minus_weight_times_y1; + a1[2] = -weight; + a1[6] = weight_times_x2 * x1; + a1[7] = weight_times_x2 * y1; + + a2[3] = minus_weight_times_x1; + a2[4] = minus_weight_times_y1; + a2[5] = -weight; + a2[6] = weight_times_y2 * x1; + a2[7] = weight_times_y2 * y1; + + for (int j = 0; j < 8; j++) { + for (int z = j; z < 8; z++) + AtAb[j * 9 + z] += a1[j]*a1[z] + a2[j]*a2[z]; + AtAb[j * 9 + 8] += a1[j]*weight_times_x2 + a2[j]*weight_times_y2; + } + } + } + for (int j = 1; j < 8; j++) + for (int z = 0; z < j; z++) + AtAb[j*9+z] = AtAb[z*9+j]; + if (!Math::eliminateUpperTriangular(AtAb, 8, 9)) + return 0; + H = Mat_(3,3); + auto * h = (double *) H.data; + h[8] = 1.; + const int m = 8, n = 9; + // start from the last row + for (int i = m-1; i >= 0; i--) { + double acc = 0; + for (int j = i+1; j < n; j++) + acc -= AtAb[i*n+j]*h[j]; + h[i] = acc / AtAb[i*n+i]; + if (std::isnan(h[i])) + return 0; // numerical imprecision + } + } else { + double a1[9] = {0, 0, -1, 0, 0, 0, 0, 0, 0}, + a2[9] = {0, 0, 0, 0, 0, -1, 0, 0, 0}, AtA[81] = {0}; + if (weights.empty()) { + for (int i = 0; i < sample_size; i++) { + const int smpl = do_norm ? 4*i : 4*sample[i]; + const auto x1 = npts[smpl], y1 = npts[smpl+1], x2 = npts[smpl+2], y2 = npts[smpl+3]; + + a1[0] = -x1; + a1[1] = -y1; + a1[6] = x2*x1; + a1[7] = x2*y1; + a1[8] = x2; + + a2[3] = -x1; + a2[4] = -y1; + a2[6] = y2*x1; + a2[7] = y2*y1; + a2[8] = y2; + + for (int j = 0; j < 9; j++) + for (int z = j; z < 9; z++) + AtA[j*9+z] += a1[j]*a1[z] + a2[j]*a2[z]; + } + } else { // use weights + for (int i = 0; i < sample_size; i++) { + const double weight = weights[i]; + if (weight < FLT_EPSILON) continue; + const int smpl = do_norm ? 4*i : 4*sample[i]; + const auto x1 = npts[smpl], y1 = npts[smpl+1], x2 = npts[smpl+2], y2 = npts[smpl+3]; + const double minus_weight_times_x1 = -weight * x1, + minus_weight_times_y1 = -weight * y1, + weight_times_x2 = weight * x2, + weight_times_y2 = weight * y2; + + a1[0] = minus_weight_times_x1; + a1[1] = minus_weight_times_y1; + a1[2] = -weight; + a1[6] = weight_times_x2 * x1; + a1[7] = weight_times_x2 * y1; + a1[8] = weight_times_x2; + + a2[3] = minus_weight_times_x1; + a2[4] = minus_weight_times_y1; + a2[5] = -weight; + a2[6] = weight_times_y2 * x1; + a2[7] = weight_times_y2 * y1; + a2[8] = weight_times_y2; + + for (int j = 0; j < 9; j++) + for (int z = j; z < 9; z++) + AtA[j*9+z] += a1[j]*a1[z] + a2[j]*a2[z]; + } + } + // copy symmetric part of covariance matrix + for (int j = 1; j < 9; j++) + for (int z = 0; z < j; z++) + AtA[j*9+z] = AtA[z*9+j]; + +#ifdef HAVE_EIGEN + H = Mat_(3,3); + // extract the last null-vector + Eigen::Map>((double *)H.data) = Eigen::Matrix + (Eigen::HouseholderQR> ( + (Eigen::Matrix (AtA))).householderQ()).col(8); +#else + Matx Vt; + Vec D; + if (! eigen(Matx(AtA), D, Vt)) return 0; + H = Mat_(3, 3, Vt.val + 72/*=8*9*/); +#endif + } + const auto * const h = (double *) H.data; + const auto * const t1 = do_norm ? T1.val : _T1.val, * const t2 = do_norm ? T2.val : _T2.val; + // H = T2^-1 H T1 + models = std::vector{ Mat(Matx33d(t1[0]*(h[0]/t2[0] - (h[6]*t2[2])/t2[0]), + t1[0]*(h[1]/t2[0] - (h[7]*t2[2])/t2[0]), h[2]/t2[0] + t1[2]*(h[0]/t2[0] - + (h[6]*t2[2])/t2[0]) + t1[5]*(h[1]/t2[0] - (h[7]*t2[2])/t2[0]) - (h[8]*t2[2])/t2[0], + t1[0]*(h[3]/t2[0] - (h[6]*t2[5])/t2[0]), t1[0]*(h[4]/t2[0] - (h[7]*t2[5])/t2[0]), + h[5]/t2[0] + t1[2]*(h[3]/t2[0] - (h[6]*t2[5])/t2[0]) + t1[5]*(h[4]/t2[0] - + (h[7]*t2[5])/t2[0]) - (h[8]*t2[5])/t2[0], t1[0]*h[6], t1[0]*h[7], + h[8] + h[6]*t1[2] + h[7]*t1[5])) }; + return 1; + } + int estimate (const std::vector &/*mask*/, std::vector &/*models*/, + const std::vector &/*weights*/) override { + return 0; + } + int getMinimumRequiredSampleSize() const override { return 4; } + int getMaxNumberOfSolutions () const override { return 1; } + void enforceRankConstraint (bool /*enforce*/) override {} +}; +Ptr HomographyNonMinimalSolver::create(const Mat &points_, bool use_ge_) { + return makePtr(points_, use_ge_); +} +Ptr HomographyNonMinimalSolver::create(const Mat &points_, const Matx33d &T1, const Matx33d &T2, bool use_ge) { + return makePtr(points_, T1, T2, use_ge); +} + +class CovarianceHomographySolverImpl : public CovarianceHomographySolver { +private: + Mat norm_pts; + Matx33d T1, T2; + float * norm_points; + std::vector mask; + int points_size; + double covariance[81] = {0}, * t1, * t2; +public: + explicit CovarianceHomographySolverImpl (const Mat &norm_points_, const Matx33d &T1_, const Matx33d &T2_) + : norm_pts(norm_points_), T1(T1_), T2(T2_) { + points_size = norm_points_.rows; + norm_points = (float *) norm_pts.data; + t1 = T1.val; t2 = T2.val; + mask = std::vector(points_size, false); + } + explicit CovarianceHomographySolverImpl (const Mat &points_) { + points_size = points_.rows; + // normalize points + std::vector sample(points_size); + for (int i = 0; i < points_size; i++) sample[i] = i; + const Ptr normTr = NormTransform::create(points_); + normTr->getNormTransformation(norm_pts, sample, points_size, T1, T2); + norm_points = (float *) norm_pts.data; + t1 = T1.val; t2 = T2.val; + mask = std::vector(points_size, false); + } + void reset () override { + // reset covariance matrix to zero and mask to false + std::fill(covariance, covariance+81, 0); + std::fill(mask.begin(), mask.end(), false); + } + + /* + * Find homography using 4-point algorithm with covariance matrix and PCA + */ + int estimate (const std::vector &new_mask, std::vector &models, + const std::vector &/*weights*/) override { double a1[9] = {0, 0, -1, 0, 0, 0, 0, 0, 0}, - a2[9] = {0, 0, 0, 0, 0, -1, 0, 0, 0}, - AtA[81] = {0}; + a2[9] = {0, 0, 0, 0, 0, -1, 0, 0, 0}; - if (weights.empty()) { - for (int i = 0; i < sample_size; i++) { + for (int i = 0; i < points_size; i++) { + if (mask[i] != new_mask[i]) { const int smpl = 4*i; const double x1 = norm_points[smpl ], y1 = norm_points[smpl+1], x2 = norm_points[smpl+2], y2 = norm_points[smpl+3]; @@ -123,71 +333,57 @@ public: a2[7] = y2*y1; a2[8] = y2; - for (int j = 0; j < 9; j++) - for (int z = j; z < 9; z++) - AtA[j*9+z] += a1[j]*a1[z] + a2[j]*a2[z]; - } - } else { - for (int i = 0; i < sample_size; i++) { - const int smpl = 4*i; - const double weight = weights[i]; - const double x1 = norm_points[smpl ], y1 = norm_points[smpl+1], - x2 = norm_points[smpl+2], y2 = norm_points[smpl+3]; - const double minus_weight_times_x1 = -weight * x1, - minus_weight_times_y1 = -weight * y1, - weight_times_x2 = weight * x2, - weight_times_y2 = weight * y2; - - a1[0] = minus_weight_times_x1; - a1[1] = minus_weight_times_y1; - a1[2] = -weight; - a1[6] = weight_times_x2 * x1; - a1[7] = weight_times_x2 * y1; - a1[8] = weight_times_x2; - - a2[3] = minus_weight_times_x1; - a2[4] = minus_weight_times_y1; - a2[5] = -weight; - a2[6] = weight_times_y2 * x1; - a2[7] = weight_times_y2 * y1; - a2[8] = weight_times_y2; - - for (int j = 0; j < 9; j++) - for (int z = j; z < 9; z++) - AtA[j*9+z] += a1[j]*a1[z] + a2[j]*a2[z]; + if (mask[i]) // if mask[i] is true then new_mask[i] must be false + for (int j = 0; j < 9; j++) + for (int z = j; z < 9; z++) + covariance[j*9+z] +=-a1[j]*a1[z] - a2[j]*a2[z]; + else + for (int j = 0; j < 9; j++) + for (int z = j; z < 9; z++) + covariance[j*9+z] += a1[j]*a1[z] + a2[j]*a2[z]; } } + mask = new_mask; // copy symmetric part of covariance matrix for (int j = 1; j < 9; j++) for (int z = 0; z < j; z++) - AtA[j*9+z] = AtA[z*9+j]; + covariance[j*9+z] = covariance[z*9+j]; #ifdef HAVE_EIGEN Mat H = Mat_(3,3); - Eigen::HouseholderQR> qr((Eigen::Matrix (AtA))); - const Eigen::Matrix &Q = qr.householderQ(); - // extract the last nullspace - Eigen::Map>((double *)H.data) = Q.col(8); + // extract the last null-vector + Eigen::Map>((double *)H.data) = Eigen::Matrix + (Eigen::HouseholderQR> ( + (Eigen::Matrix (covariance))).householderQ()).col(8); #else - Matx Vt; - Vec D; - if (! eigen(Matx(AtA), D, Vt)) return 0; - Mat H = Mat_(3, 3, Vt.val + 72/*=8*9*/); + Matx Vt; + Vec D; + if (! eigen(Matx(covariance), D, Vt)) return 0; + Mat H = Mat_(3, 3, Vt.val + 72/*=8*9*/); #endif - models = std::vector{ T2.inv() * H * T1 }; + const auto * const h = (double *) H.data; + // H = T2^-1 H T1 + models = std::vector{ Mat(Matx33d(t1[0]*(h[0]/t2[0] - (h[6]*t2[2])/t2[0]), + t1[0]*(h[1]/t2[0] - (h[7]*t2[2])/t2[0]), h[2]/t2[0] + t1[2]*(h[0]/t2[0] - + (h[6]*t2[2])/t2[0]) + t1[5]*(h[1]/t2[0] - (h[7]*t2[2])/t2[0]) - (h[8]*t2[2])/t2[0], + t1[0]*(h[3]/t2[0] - (h[6]*t2[5])/t2[0]), t1[0]*(h[4]/t2[0] - (h[7]*t2[5])/t2[0]), + h[5]/t2[0] + t1[2]*(h[3]/t2[0] - (h[6]*t2[5])/t2[0]) + t1[5]*(h[4]/t2[0] - + (h[7]*t2[5])/t2[0]) - (h[8]*t2[5])/t2[0], t1[0]*h[6], t1[0]*h[7], + h[8] + h[6]*t1[2] + h[7]*t1[5])) }; + return 1; } - + void enforceRankConstraint (bool /*enforce*/) override {} int getMinimumRequiredSampleSize() const override { return 4; } int getMaxNumberOfSolutions () const override { return 1; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; -Ptr HomographyNonMinimalSolver::create(const Mat &points_) { - return makePtr(points_); +Ptr CovarianceHomographySolver::create (const Mat &points) { + return makePtr(points); +} +Ptr CovarianceHomographySolver::create (const Mat &points, const Matx33d &T1, const Matx33d &T2) { + return makePtr(points, T1, T2); } class AffineMinimalSolverImpl : public AffineMinimalSolver { @@ -196,7 +392,7 @@ private: const float * const points; public: explicit AffineMinimalSolverImpl (const Mat &points_) : - points_mat(&points_), points((float *) points_.data) {} + points_mat(&points_), points((float *) points_mat->data) {} /* Affine transformation x1 y1 1 0 0 0 a u1 @@ -232,9 +428,6 @@ public: } int getSampleSize() const override { return 3; } int getMaxNumberOfSolutions () const override { return 1; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; Ptr AffineMinimalSolver::create(const Mat &points_) { return makePtr(points_); @@ -243,22 +436,32 @@ Ptr AffineMinimalSolver::create(const Mat &points_) { class AffineNonMinimalSolverImpl : public AffineNonMinimalSolver { private: const Mat * points_mat; - const float * const points; - // const NormTransform norm_transform; + Ptr normTr; + Matx33d _T1, _T2; + bool do_norm; public: - explicit AffineNonMinimalSolverImpl (const Mat &points_) : - points_mat(&points_), points((float*) points_.data) - /*, norm_transform(points_)*/ {} + explicit AffineNonMinimalSolverImpl (const Mat &points_, InputArray T1, InputArray T2) : + points_mat(&points_) { + if (!T1.empty() && !T2.empty()) { + do_norm = false; + _T1 = T1.getMat(); + _T2 = T2.getMat(); + } else { + do_norm = true; + normTr = NormTransform::create(points_); + } + } int estimate (const std::vector &sample, int sample_size, std::vector &models, const std::vector &weights) const override { - // surprisingly normalization of points does not improve the output model - // Mat norm_points_, T1, T2; - // norm_transform.getNormTransformation(norm_points_, sample, sample_size, T1, T2); - // const auto * const n_pts = (double *) norm_points_.data; - if (sample_size < getMinimumRequiredSampleSize()) return 0; + Matx33d T1, T2; + Mat norm_points_; + if (do_norm) + normTr->getNormTransformation(norm_points_, sample, sample_size, T1, T2); + const auto * const pts = normTr ? (float *) norm_points_.data : (float *) points_mat->data; + // do Least Squares // Ax = b -> A^T Ax = A^T b // x = (A^T A)^-1 A^T b @@ -268,12 +471,8 @@ public: if (weights.empty()) for (int p = 0; p < sample_size; p++) { - // if (weights != nullptr) weight = weights[sample[p]]; - - const int smpl = 4*sample[p]; - const double x1=points[smpl], y1=points[smpl+1], x2=points[smpl+2], y2=points[smpl+3]; - // const double x1=n_pts[smpl], y1=n_pts[smpl+1], x2=n_pts[smpl+2], y2=n_pts[smpl+3]; - + const int idx = do_norm ? 4*p : 4*sample[p]; + const auto x1=pts[idx], y1=pts[idx+1], x2=pts[idx+2], y2=pts[idx+3]; r1[0] = x1; r1[1] = y1; @@ -288,12 +487,13 @@ public: } else for (int p = 0; p < sample_size; p++) { - const int smpl = 4*sample[p]; - const double weight = weights[p]; - const double weight_times_x1 = weight * points[smpl ], - weight_times_y1 = weight * points[smpl+1], - weight_times_x2 = weight * points[smpl+2], - weight_times_y2 = weight * points[smpl+3]; + const auto weight = weights[p]; + if (weight < FLT_EPSILON) continue; + const int idx = do_norm ? 4*p : 4*sample[p]; + const double weight_times_x1 = weight * pts[idx ], + weight_times_y1 = weight * pts[idx+1], + weight_times_x2 = weight * pts[idx+2], + weight_times_y2 = weight * pts[idx+3]; r1[0] = weight_times_x1; r1[1] = weight_times_y1; @@ -318,21 +518,126 @@ public: Vec6d aff; if (!solve(Matx66d(AtA), Vec6d(Ab), aff)) return 0; - models[0] = Mat(Matx33d(aff(0), aff(1), aff(2), - aff(3), aff(4), aff(5), - 0, 0, 1)); - - // models[0] = T2.inv() * models[0] * T1; + const double h[9] = {aff(0), aff(1), aff(2), + aff(3), aff(4), aff(5), + 0, 0, 1}; + const auto * const t1 = normTr ? T1.val : _T1.val, * const t2 = normTr ? T2.val : _T2.val; + // A = T2^-1 A T1 + models = std::vector{ Mat(Matx33d(t1[0]*(h[0]/t2[0] - (h[6]*t2[2])/t2[0]), + t1[0]*(h[1]/t2[0] - (h[7]*t2[2])/t2[0]), h[2]/t2[0] + t1[2]*(h[0]/t2[0] - + (h[6]*t2[2])/t2[0]) + t1[5]*(h[1]/t2[0] - (h[7]*t2[2])/t2[0]) - (h[8]*t2[2])/t2[0], + t1[0]*(h[3]/t2[0] - (h[6]*t2[5])/t2[0]), t1[0]*(h[4]/t2[0] - (h[7]*t2[5])/t2[0]), + h[5]/t2[0] + t1[2]*(h[3]/t2[0] - (h[6]*t2[5])/t2[0]) + t1[5]*(h[4]/t2[0] - + (h[7]*t2[5])/t2[0]) - (h[8]*t2[5])/t2[0], t1[0]*h[6], t1[0]*h[7], + h[8] + h[6]*t1[2] + h[7]*t1[5])) }; return 1; } + int estimate (const std::vector &/*mask*/, std::vector &/*models*/, + const std::vector &/*weights*/) override { + return 0; + } + void enforceRankConstraint (bool /*enforce*/) override {} int getMinimumRequiredSampleSize() const override { return 3; } int getMaxNumberOfSolutions () const override { return 1; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; -Ptr AffineNonMinimalSolver::create(const Mat &points_) { - return makePtr(points_); +Ptr AffineNonMinimalSolver::create(const Mat &points_, InputArray T1, InputArray T2) { + return makePtr(points_, T1, T2); +} + +class CovarianceAffineSolverImpl : public CovarianceAffineSolver { +private: + Mat norm_pts; + Matx33d T1, T2; + float * norm_points; + std::vector mask; + int points_size; + double covariance[36] = {0}, Ab[6] = {0}, * t1, * t2; +public: + explicit CovarianceAffineSolverImpl (const Mat &norm_points_, const Matx33d &T1_, const Matx33d &T2_) + : norm_pts(norm_points_), T1(T1_), T2(T2_) { + points_size = norm_points_.rows; + norm_points = (float *) norm_pts.data; + t1 = T1.val; t2 = T2.val; + mask = std::vector(points_size, false); + } + explicit CovarianceAffineSolverImpl (const Mat &points_) { + points_size = points_.rows; + // normalize points + std::vector sample(points_size); + for (int i = 0; i < points_size; i++) sample[i] = i; + const Ptr normTr = NormTransform::create(points_); + normTr->getNormTransformation(norm_pts, sample, points_size, T1, T2); + norm_points = (float *) norm_pts.data; + t1 = T1.val; t2 = T2.val; + mask = std::vector(points_size, false); + } + void reset () override { + std::fill(covariance, covariance+36, 0); + std::fill(Ab, Ab+6, 0); + std::fill(mask.begin(), mask.end(), false); + } + /* + * Find affine transformation using linear method with covariance matrix and PCA + */ + int estimate (const std::vector &new_mask, std::vector &models, + const std::vector &) override { + double r1[6] = {0, 0, 1, 0, 0, 0}; // row 1 of A + double r2[6] = {0, 0, 0, 0, 0, 1}; // row 2 of A + for (int i = 0; i < points_size; i++) { + if (mask[i] != new_mask[i]) { + const int smpl = 4*i; + const double x1 = norm_points[smpl ], y1 = norm_points[smpl+1], + x2 = norm_points[smpl+2], y2 = norm_points[smpl+3]; + + r1[0] = x1; + r1[1] = y1; + + r2[3] = x1; + r2[4] = y1; + + if (mask[i]) // if mask[i] is true then new_mask[i] must be false + for (int j = 0; j < 6; j++) { + for (int z = j; z < 6; z++) + covariance[j*6+z] +=-r1[j]*r1[z] - r2[j]*r2[z]; + Ab[j] +=-r1[j]*x2 - r2[j]*y2; + } + else + for (int j = 0; j < 6; j++) { + for (int z = j; z < 6; z++) + covariance[j*6+z] += r1[j]*r1[z] + r2[j]*r2[z]; + Ab[j] += r1[j]*x2 + r2[j]*y2; + } + } + } + mask = new_mask; + + // copy symmetric part of covariance matrix + for (int j = 1; j < 6; j++) + for (int z = 0; z < j; z++) + covariance[j*6+z] = covariance[z*6+j]; + + Vec6d aff; + if (!solve(Matx66d(covariance), Vec6d(Ab), aff)) + return 0; + double a[9] = { aff(0), aff(1), aff(2), aff(3), aff(4), aff(5), 0, 0, 1 }; + models = std::vector{ Mat(Matx33d(t1[0]*(a[0]/t2[0] - (a[6]*t2[2])/t2[0]), + t1[0]*(a[1]/t2[0] - (a[7]*t2[2])/t2[0]), a[2]/t2[0] + t1[2]*(a[0]/t2[0] - + (a[6]*t2[2])/t2[0]) + t1[5]*(a[1]/t2[0] - (a[7]*t2[2])/t2[0]) - (a[8]*t2[2])/t2[0], + t1[0]*(a[3]/t2[0] - (a[6]*t2[5])/t2[0]), t1[0]*(a[4]/t2[0] - (a[7]*t2[5])/t2[0]), + a[5]/t2[0] + t1[2]*(a[3]/t2[0] - (a[6]*t2[5])/t2[0]) + t1[5]*(a[4]/t2[0] - + (a[7]*t2[5])/t2[0]) - (a[8]*t2[5])/t2[0], t1[0]*a[6], t1[0]*a[7], + a[8] + a[6]*t1[2] + a[7]*t1[5])) }; + return 1; + } + void enforceRankConstraint (bool /*enforce*/) override {} + int getMinimumRequiredSampleSize() const override { return 3; } + int getMaxNumberOfSolutions () const override { return 1; } +}; +Ptr CovarianceAffineSolver::create (const Mat &points, const Matx33d &T1, const Matx33d &T2) { + return makePtr(points, T1, T2); +} +Ptr CovarianceAffineSolver::create (const Mat &points) { + return makePtr(points); } }} \ No newline at end of file diff --git a/modules/calib3d/src/usac/local_optimization.cpp b/modules/calib3d/src/usac/local_optimization.cpp index 3964edec9a..8c3703940f 100644 --- a/modules/calib3d/src/usac/local_optimization.cpp +++ b/modules/calib3d/src/usac/local_optimization.cpp @@ -22,14 +22,18 @@ protected: std::vector energies, weights; std::set used_edges; std::vector gc_models; + + Ptr termination; + int num_lo_optimizations = 0, current_ransac_iter = 0; public: + void setCurrentRANSACiter (int ransac_iter) override { current_ransac_iter = ransac_iter; } // In lo_sampler_ the sample size should be set and be equal gc_sample_size_ - GraphCutImpl (const Ptr &estimator_, const Ptr &error_, const Ptr &quality_, + GraphCutImpl (const Ptr &estimator_, const Ptr &quality_, const Ptr &neighborhood_graph_, const Ptr &lo_sampler_, - double threshold_, double spatial_coherence_term, int gc_inner_iteration_number_) : + double threshold_, double spatial_coherence_term, int gc_inner_iteration_number_, Ptr termination_) : neighborhood_graph (neighborhood_graph_), estimator (estimator_), quality (quality_), - lo_sampler (lo_sampler_), error (error_) { + lo_sampler (lo_sampler_), error (quality_->getErrorFnc()), termination(termination_) { points_size = quality_->getPointsSize(); spatial_coherence = spatial_coherence_term; @@ -81,9 +85,13 @@ public: gc_models[model_idx].copyTo(new_model); } } + + if (termination != nullptr && is_best_model_updated && current_ransac_iter > termination->update(best_model, best_model_score.inlier_number)) { + is_best_model_updated = false; // to break outer loop + } + } // end of inner GC local optimization } // end of while loop - return true; } @@ -91,7 +99,6 @@ private: // find inliers using graph cut algorithm. int labeling (const Mat& model) { const auto &errors = error->getErrors(model); - detail::GCGraph graph; for (int pt = 0; pt < points_size; pt++) @@ -151,10 +158,8 @@ private: has_edges = true; } } - - if (!has_edges) + if (! has_edges) return quality->getInliers(model, labeling_inliers); - graph.maxFlow(); int inlier_number = 0; @@ -163,23 +168,17 @@ private: labeling_inliers[inlier_number++] = pt; return inlier_number; } - Ptr clone(int state) const override { - return makePtr(estimator->clone(), error->clone(), quality->clone(), - neighborhood_graph,lo_sampler->clone(state), sqr_trunc_thr / 2.25, - spatial_coherence, lo_inner_iterations); - } + int getNumLOoptimizations () const override { return num_lo_optimizations; } }; -Ptr GraphCut::create(const Ptr &estimator_, const Ptr &error_, +Ptr GraphCut::create(const Ptr &estimator_, const Ptr &quality_, const Ptr &neighborhood_graph_, const Ptr &lo_sampler_, double threshold_, - double spatial_coherence_term, int gc_inner_iteration_number) { - return makePtr(estimator_, error_, quality_, neighborhood_graph_, lo_sampler_, - threshold_, spatial_coherence_term, gc_inner_iteration_number); + double spatial_coherence_term, int gc_inner_iteration_number, Ptr termination_) { + return makePtr(estimator_, quality_, neighborhood_graph_, lo_sampler_, + threshold_, spatial_coherence_term, gc_inner_iteration_number, termination_); } -/* -* http://cmp.felk.cvut.cz/~matas/papers/chum-dagm03.pdf -*/ +// http://cmp.felk.cvut.cz/~matas/papers/chum-dagm03.pdf class InnerIterativeLocalOptimizationImpl : public InnerIterativeLocalOptimization { private: const Ptr estimator; @@ -197,23 +196,18 @@ private: double threshold, new_threshold, threshold_step; std::vector weights; public: - InnerIterativeLocalOptimizationImpl (const Ptr &estimator_, const Ptr &quality_, const Ptr &lo_sampler_, int pts_size, double threshold_, bool is_iterative_, int lo_iter_sample_size_, int lo_inner_iterations_=10, int lo_iter_max_iterations_=5, double threshold_multiplier_=4) : estimator (estimator_), quality (quality_), lo_sampler (lo_sampler_) - , lo_iter_sample_size(0) - , new_threshold(0), threshold_step(0) - { + , lo_iter_sample_size(0), new_threshold(0), threshold_step(0) { lo_inner_max_iterations = lo_inner_iterations_; lo_iter_max_iterations = lo_iter_max_iterations_; threshold = threshold_; - lo_sample_size = lo_sampler->getSubsetSize(); - is_iterative = is_iterative_; if (is_iterative) { lo_iter_sample_size = lo_iter_sample_size_; @@ -225,7 +219,6 @@ public: // In the last iteration there be original threshold θ. threshold_step = (new_threshold - threshold) / lo_iter_max_iterations_; } - lo_models = std::vector(estimator->getMaxNumSolutionsNonMinimal()); // Allocate max memory to avoid reallocation @@ -327,12 +320,6 @@ public: } return true; } - Ptr clone(int state) const override { - return makePtr(estimator->clone(), quality->clone(), - lo_sampler->clone(state),(int)inliers_of_best_model.size(), threshold, is_iterative, - lo_iter_sample_size, lo_inner_max_iterations, lo_iter_max_iterations, - new_threshold / threshold); - } }; Ptr InnerIterativeLocalOptimization::create (const Ptr &estimator_, const Ptr &quality_, @@ -345,293 +332,275 @@ Ptr InnerIterativeLocalOptimization::create lo_inner_iterations_, lo_iter_max_iterations_, threshold_multiplier_); } -class SigmaConsensusImpl : public SigmaConsensus { +class SimpleLocalOptimizationImpl : public SimpleLocalOptimization { private: - const Ptr estimator; const Ptr quality; const Ptr error; - const Ptr verifier; - const GammaValues& gamma_generator; - // The degrees of freedom of the data from which the model is estimated. - // E.g., for models coming from point correspondences (x1,y1,x2,y2), it is 4. - const int degrees_of_freedom; - // A 0.99 quantile of the Chi^2-distribution to convert sigma values to residuals - const double k; - // Calculating (DoF - 1) / 2 which will be used for the estimation and, - // due to being constant, it is better to calculate it a priori. - double dof_minus_one_per_two; - const double C; - // The size of a minimal sample used for the estimation - const int sample_size; - // Calculating 2^(DoF - 1) which will be used for the estimation and, - // due to being constant, it is better to calculate it a priori. - double two_ad_dof; - // Calculating C * 2^(DoF - 1) which will be used for the estimation and, - // due to being constant, it is better to calculate it a priori. - double C_times_two_ad_dof; - // Calculating the gamma value of (DoF - 1) / 2 which will be used for the estimation and, - // due to being constant, it is better to calculate it a priori. - double squared_sigma_max_2, one_over_sigma; - // Calculating the upper incomplete gamma value of (DoF - 1) / 2 with k^2 / 2. - const double gamma_k; - // Calculating the lower incomplete gamma value of (DoF - 1) / 2 which will be used for the estimation and, - // due to being constant, it is better to calculate it a priori. - double max_sigma_sqr; - const int points_size, number_of_irwls_iters; - const double maximum_threshold, max_sigma; - - std::vector sqr_residuals, sigma_weights; - std::vector sqr_residuals_idxs; - // Models fit by weighted least-squares fitting - std::vector sigma_models; - // Points used in the weighted least-squares fitting - std::vector sigma_inliers; - // Weights used in the the weighted least-squares fitting - int max_lo_sample_size, stored_gamma_number_min1; - double scale_of_stored_gammas; - RNG rng; - const std::vector &stored_gamma_values; + const Ptr estimator; + const Ptr termination; + const Ptr random_generator; + const Ptr weight_fnc; + // unlike to @random_generator which has fixed subset size + // @random_generator_smaller_subset is used to draw smaller + // amount of points which depends on current number of inliers + Ptr random_generator_smaller_subset; + int points_size, max_lo_iters, non_min_sample_size, current_ransac_iter; + std::vector weights; + std::vector inliers; + std::vector models; + double inlier_threshold_sqr; + int num_lo_optimizations = 0; + bool updated_lo = false; public: - - SigmaConsensusImpl (const Ptr &estimator_, const Ptr &error_, - const Ptr &quality_, const Ptr &verifier_, - int max_lo_sample_size_, int number_of_irwls_iters_, int DoF, - double sigma_quantile, double upper_incomplete_of_sigma_quantile, double C_, - double maximum_thr) : estimator (estimator_), quality(quality_), - error (error_), verifier(verifier_), - gamma_generator(GammaValues::getSingleton()), - degrees_of_freedom(DoF), k (sigma_quantile), C(C_), - sample_size(estimator_->getMinimalSampleSize()), - gamma_k (upper_incomplete_of_sigma_quantile), points_size (quality_->getPointsSize()), - number_of_irwls_iters (number_of_irwls_iters_), - maximum_threshold(maximum_thr), max_sigma (maximum_thr), - stored_gamma_values(gamma_generator.getGammaValues()) - { - dof_minus_one_per_two = (degrees_of_freedom - 1.0) / 2.0; - two_ad_dof = std::pow(2.0, dof_minus_one_per_two); - C_times_two_ad_dof = C * two_ad_dof; - // Calculate 2 * \sigma_{max}^2 a priori - squared_sigma_max_2 = max_sigma * max_sigma * 2.0; - // Divide C * 2^(DoF - 1) by \sigma_{max} a priori - one_over_sigma = C_times_two_ad_dof / max_sigma; - max_sigma_sqr = squared_sigma_max_2 * 0.5; - sqr_residuals = std::vector(points_size); - sqr_residuals_idxs = std::vector(points_size); - sigma_inliers = std::vector(points_size); - max_lo_sample_size = max_lo_sample_size_; - sigma_weights = std::vector(points_size); - sigma_models = std::vector(estimator->getMaxNumSolutionsNonMinimal()); - stored_gamma_number_min1 = gamma_generator.getTableSize()-1; - scale_of_stored_gammas = gamma_generator.getScaleOfGammaValues(); + SimpleLocalOptimizationImpl (const Ptr &quality_, const Ptr &estimator_, + const Ptr termination_, const Ptr &random_gen, Ptr weight_fnc_, + int max_lo_iters_, double inlier_threshold_sqr_, bool update_lo_) : + quality(quality_), error(quality_->getErrorFnc()), estimator(estimator_), termination(termination_), + random_generator(random_gen), weight_fnc(weight_fnc_) { + max_lo_iters = max_lo_iters_; + non_min_sample_size = random_generator->getSubsetSize(); + current_ransac_iter = 0; + inliers = std::vector(quality_->getPointsSize()); + models = std::vector(estimator_->getMaxNumberOfSolutions()); + points_size = quality_->getPointsSize(); + inlier_threshold_sqr = inlier_threshold_sqr_; + if (weight_fnc != nullptr) weights = std::vector(points_size); + random_generator_smaller_subset = nullptr; + updated_lo = update_lo_; } + void setCurrentRANSACiter (int ransac_iter) override { current_ransac_iter = ransac_iter; } + int getNumLOoptimizations () const override { return num_lo_optimizations; } + bool refineModel (const Mat &best_model, const Score &best_model_score, Mat &new_model, Score &new_model_score) override { + new_model_score = best_model_score; + best_model.copyTo(new_model); - // https://github.com/danini/magsac - bool refineModel (const Mat &in_model, const Score &best_model_score, - Mat &new_model, Score &new_model_score) override { - int residual_cnt = 0; + int num_inliers; + if (weights.empty()) + num_inliers = Quality::getInliers(error, best_model, inliers, inlier_threshold_sqr); + else num_inliers = weight_fnc->getInliersWeights(error->getErrors(best_model), inliers, weights); + auto update_generator = [&] (int num_inls) { + if (num_inls <= non_min_sample_size) { + // add new random generator if number of inliers is fewer than non-minimal sample size + const int new_sample_size = (int)(0.6*num_inls); + if (new_sample_size <= estimator->getMinimumRequiredSampleSize()) + return false; + if (random_generator_smaller_subset == nullptr) + random_generator_smaller_subset = UniformRandomGenerator::create(num_inls/*state*/, quality->getPointsSize(), new_sample_size); + else random_generator_smaller_subset->setSubsetSize(new_sample_size); + } + return true; + }; + if (!update_generator(num_inliers)) + return false; - if (verifier->isModelGood(in_model)) { - if (verifier->hasErrors()) { - const std::vector &errors = verifier->getErrors(); - for (int point_idx = 0; point_idx < points_size; ++point_idx) { - // Calculate the residual of the current point - const auto residual = sqrtf(errors[point_idx]); - if (max_sigma > residual) { - // Store the residual of the current point and its index - sqr_residuals[residual_cnt] = residual; - sqr_residuals_idxs[residual_cnt++] = point_idx; + int max_lo_iters_ = max_lo_iters, last_update = 0, last_inliers_update = 0; + for (int iter = 0; iter < max_lo_iters_; iter++) { + int num_models; + if (num_inliers <= non_min_sample_size) + num_models = estimator->estimate(new_model, random_generator_smaller_subset->generateUniqueRandomSubset(inliers, num_inliers), + random_generator_smaller_subset->getSubsetSize(), models, weights); + else num_models = estimator->estimate(new_model, random_generator->generateUniqueRandomSubset(inliers, num_inliers), non_min_sample_size, models, weights); + for (int m = 0; m < num_models; m++) { + const auto score = quality->getScore(models[m]); + if (score.isBetter(new_model_score)) { + last_update = iter; last_inliers_update = new_model_score.inlier_number - score.inlier_number; + if (updated_lo) { + if (max_lo_iters_ < iter + 5 && last_inliers_update >= 1) { + max_lo_iters_ = iter + 5; + } + } + models[m].copyTo(new_model); + new_model_score = score; + if (termination != nullptr && current_ransac_iter > termination->update(new_model, new_model_score.inlier_number)) + return true; // terminate LO if max number of iterations reached + if (score.inlier_number >= best_model_score.inlier_number || + score.inlier_number > non_min_sample_size) { + // update inliers if new model has more than previous best model + if (weights.empty()) + num_inliers = Quality::getInliers(error, best_model, inliers, inlier_threshold_sqr); + else num_inliers = weight_fnc->getInliersWeights(error->getErrors(best_model), inliers, weights); + if (!update_generator(num_inliers)) + return true; } - // Interrupt if there is no chance of being better - if (residual_cnt + points_size - point_idx < best_model_score.inlier_number) - return false; - } - } else { - error->setModelParameters(in_model); - - for (int point_idx = 0; point_idx < points_size; ++point_idx) { - const double sqr_residual = error->getError(point_idx); - if (sqr_residual < max_sigma_sqr) { - // Store the residual of the current point and its index - sqr_residuals[residual_cnt] = sqr_residual; - sqr_residuals_idxs[residual_cnt++] = point_idx; - } - - if (residual_cnt + points_size - point_idx < best_model_score.inlier_number) - return false; } } - } else return false; - - in_model.copyTo(new_model); - new_model_score = Score(); - - // Do the iteratively re-weighted least squares fitting - for (int iterations = 0; iterations < number_of_irwls_iters; iterations++) { - int sigma_inliers_cnt = 0; - // If the current iteration is not the first, the set of possibly inliers - // (i.e., points closer than the maximum threshold) have to be recalculated. - if (iterations > 0) { - // error->setModelParameters(polished_model); - error->setModelParameters(new_model); - // Remove everything from the residual vector - residual_cnt = 0; - - // Collect the points which are closer than the maximum threshold - for (int point_idx = 0; point_idx < points_size; ++point_idx) { - // Calculate the residual of the current point - const double sqr_residual = error->getError(point_idx); - if (sqr_residual < max_sigma_sqr) { - // Store the residual of the current point and its index - sqr_residuals[residual_cnt] = sqr_residual; - sqr_residuals_idxs[residual_cnt++] = point_idx; - } - } - sigma_inliers_cnt = 0; - } - - // Calculate the weight of each point - for (int i = 0; i < residual_cnt; i++) { - // Get the position of the gamma value in the lookup table - int x = (int)round(scale_of_stored_gammas * sqr_residuals[i] - / squared_sigma_max_2); - - // If the sought gamma value is not stored in the lookup, return the closest element - if (x >= stored_gamma_number_min1 || x < 0 /*overflow*/) // actual number of gamma values is 1 more, so >= - x = stored_gamma_number_min1; - - sigma_inliers[sigma_inliers_cnt] = sqr_residuals_idxs[i]; // store index of point for LSQ - sigma_weights[sigma_inliers_cnt++] = one_over_sigma * (stored_gamma_values[x] - gamma_k); - } - - // random shuffle sigma inliers - if (sigma_inliers_cnt > max_lo_sample_size) - for (int i = sigma_inliers_cnt-1; i > 0; i--) { - const int idx = rng.uniform(0, i+1); - std::swap(sigma_inliers[i], sigma_inliers[idx]); - std::swap(sigma_weights[i], sigma_weights[idx]); - } - const int num_est_models = estimator->estimateModelNonMinimalSample - (sigma_inliers, std::min(max_lo_sample_size, sigma_inliers_cnt), - sigma_models, sigma_weights); - - if (num_est_models == 0) - break; // break iterations - - // Update the model parameters - Mat polished_model = sigma_models[0]; - if (num_est_models > 1) { - // find best over other models - Score sigma_best_score = quality->getScore(polished_model); - for (int m = 1; m < num_est_models; m++) { - const Score sc = quality->getScore(sigma_models[m]); - if (sc.isBetter(sigma_best_score)) { - polished_model = sigma_models[m]; - sigma_best_score = sc; - } - } - } - - const Score polished_model_score = quality->getScore(polished_model); - if (polished_model_score.isBetter(new_model_score)){ - new_model_score = polished_model_score; - polished_model.copyTo(new_model); + if (updated_lo && iter - last_update >= 10) { + break; } } - - const Score in_model_score = quality->getScore(in_model); - if (in_model_score.isBetter(new_model_score)) { - new_model_score = in_model_score; - in_model.copyTo(new_model); - } - return true; } - Ptr clone(int state) const override { - return makePtr(estimator->clone(), error->clone(), quality->clone(), - verifier->clone(state), max_lo_sample_size, - number_of_irwls_iters, degrees_of_freedom, k, gamma_k, C, maximum_threshold); +}; +Ptr SimpleLocalOptimization::create (const Ptr &quality_, + const Ptr &estimator_, const Ptr termination_, const Ptr &random_gen, + const Ptr weight_fnc, int max_lo_iters_, double inlier_thr_sqr, bool updated_lo) { + return makePtr (quality_, estimator_, termination_, random_gen, weight_fnc, max_lo_iters_, inlier_thr_sqr, updated_lo); +} + +class MagsacWeightFunctionImpl : public MagsacWeightFunction { +private: + const std::vector &stored_gamma_values; + double C, max_sigma, max_sigma_sqr, scale_of_stored_gammas, one_over_sigma, gamma_k, squared_sigma_max_2, rescale_err; + int DoF; + unsigned int stored_gamma_number_min1; +public: + MagsacWeightFunctionImpl (const Ptr &gamma_generator, + int DoF_, double upper_incomplete_of_sigma_quantile, double C_, double max_sigma_) : + stored_gamma_values (gamma_generator->getGammaValues()) { + gamma_k = upper_incomplete_of_sigma_quantile; + stored_gamma_number_min1 = static_cast(gamma_generator->getTableSize()-1); + scale_of_stored_gammas = gamma_generator->getScaleOfGammaValues(); + DoF = DoF_; C = C_; + max_sigma = max_sigma_; + squared_sigma_max_2 = max_sigma * max_sigma * 2.0; + one_over_sigma = C * pow(2.0, (DoF - 1.0) * 0.5) / max_sigma; + max_sigma_sqr = squared_sigma_max_2 * 0.5; + rescale_err = scale_of_stored_gammas / squared_sigma_max_2; + } + int getInliersWeights (const std::vector &errors, std::vector &inliers, std::vector &weights) const override { + return getInliersWeights(errors, inliers, weights, one_over_sigma, rescale_err, max_sigma_sqr); + } + int getInliersWeights (const std::vector &errors, std::vector &inliers, std::vector &weights, double thr_sqr) const override { + const auto _max_sigma = thr_sqr; + const auto _squared_sigma_max_2 = _max_sigma * _max_sigma * 2.0; + const auto _one_over_sigma = C * pow(2.0, (DoF - 1.0) * 0.5) / _max_sigma; + const auto _max_sigma_sqr = _squared_sigma_max_2 * 0.5; + const auto _rescale_err = scale_of_stored_gammas / _squared_sigma_max_2; + return getInliersWeights(errors, inliers, weights, _one_over_sigma, _rescale_err, _max_sigma_sqr); + } + double getThreshold () const override { + return max_sigma_sqr; + } +private: + int getInliersWeights (const std::vector &errors, std::vector &inliers, std::vector &weights, + double _one_over_sigma, double _rescale_err, double _max_sigma_sqr) const { + int num_inliers = 0, p = 0; + for (const auto &e : errors) { + // Calculate the residual of the current point + if (e < _max_sigma_sqr) { + // Get the position of the gamma value in the lookup table + auto x = static_cast(_rescale_err * e); + if (x > stored_gamma_number_min1) + x = stored_gamma_number_min1; + inliers[num_inliers] = p; // store index of point for LSQ + weights[num_inliers++] = _one_over_sigma * (stored_gamma_values[x] - gamma_k); + } + p++; + } + return num_inliers; } }; -Ptr -SigmaConsensus::create(const Ptr &estimator_, const Ptr &error_, - const Ptr &quality, const Ptr &verifier_, - int max_lo_sample_size, int number_of_irwls_iters_, int DoF, - double sigma_quantile, double upper_incomplete_of_sigma_quantile, double C_, - double maximum_thr) { - return makePtr(estimator_, error_, quality, verifier_, - max_lo_sample_size, number_of_irwls_iters_, DoF, sigma_quantile, - upper_incomplete_of_sigma_quantile, C_, maximum_thr); +Ptr MagsacWeightFunction::create(const Ptr &gamma_generator_, + int DoF_, double upper_incomplete_of_sigma_quantile, double C_, double max_sigma_) { + return makePtr(gamma_generator_, DoF_, upper_incomplete_of_sigma_quantile, C_, max_sigma_); } /////////////////////////////////////////// FINAL MODEL POLISHER //////////////////////// -class LeastSquaresPolishingImpl : public LeastSquaresPolishing { +class NonMinimalPolisherImpl : public NonMinimalPolisher { private: - const Ptr estimator; const Ptr quality; - int lsq_iterations; - std::vector inliers; + const Ptr solver; + const Ptr error_fnc; + const Ptr weight_fnc; + std::vector mask, mask_best; std::vector models; std::vector weights; + std::vector errors_best; + std::vector inliers; + double threshold, iou_thr, max_thr; + int max_iters, points_size; + bool is_covariance, CHANGE_WEIGHTS = true; public: - - LeastSquaresPolishingImpl(const Ptr &estimator_, const Ptr &quality_, - int lsq_iterations_) : - estimator(estimator_), quality(quality_) { - lsq_iterations = lsq_iterations_; - // allocate memory for inliers array and models - inliers = std::vector(quality_->getPointsSize()); - models = std::vector(estimator->getMaxNumSolutionsNonMinimal()); + NonMinimalPolisherImpl (const Ptr &quality_, const Ptr &solver_, + Ptr weight_fnc_, int max_iters_, double iou_thr_) : + quality(quality_), solver(solver_), error_fnc(quality_->getErrorFnc()), weight_fnc(weight_fnc_) { + max_iters = max_iters_; + points_size = quality_->getPointsSize(); + threshold = quality_->getThreshold(); + iou_thr = iou_thr_; + is_covariance = dynamic_cast(solver_.get()) != nullptr; + mask = std::vector(points_size); + mask_best = std::vector(points_size); + inliers = std::vector(points_size); + if (weight_fnc != nullptr) { + weights = std::vector(points_size); + max_thr = weight_fnc->getThreshold(); + if (is_covariance) + CV_Error(cv::Error::StsBadArg, "Covariance polisher cannot be combined with weights!"); + } } - bool polishSoFarTheBestModel(const Mat &model, const Score &best_model_score, - Mat &new_model, Score &out_score) override { - // get inliers from input model - int inlier_number = quality->getInliers(model, inliers); - if (inlier_number < estimator->getMinimalSampleSize()) - return false; - - out_score = Score(); // set the worst case - - // several all-inlier least-squares refines model better than only one but for - // big amount of points may be too time-consuming. - for (int lsq_iter = 0; lsq_iter < lsq_iterations; lsq_iter++) { - bool model_updated = false; - - // estimate non minimal models with all inliers - const int num_models = estimator->estimateModelNonMinimalSample(inliers, - inlier_number, models, weights); - for (int model_idx = 0; model_idx < num_models; model_idx++) { - const Score score = quality->getScore(models[model_idx]); - if (best_model_score.isBetter(score)) - continue; - if (score.isBetter(out_score)) { - models[model_idx].copyTo(new_model); - out_score = score; - model_updated = true; + bool polishSoFarTheBestModel (const Mat &model, const Score &best_model_score, + Mat &new_model, Score &new_model_score) override { + int num_inliers = 0; + if (weights.empty()) { + quality->getInliers(model, mask_best); + if (!is_covariance) + for (int p = 0; p < points_size; p++) + if (mask_best[p]) inliers[num_inliers++] = p; + } else { + errors_best = error_fnc->getErrors(model); + num_inliers = weight_fnc->getInliersWeights(errors_best, inliers, weights, max_thr); + } + new_model_score = best_model_score; + model.copyTo(new_model); + int last_update = -1; + for (int iter = 0; iter < max_iters; iter++) { + int num_sols; + if (is_covariance) num_sols = solver->estimate(mask_best, models, weights); + else num_sols = solver->estimate(new_model, inliers, num_inliers, models, weights); + Score prev_score; + for (int i = 0; i < num_sols; i++) { + const auto &errors = error_fnc->getErrors(models[i]); + const auto score = quality->getScore(errors); + if (score.isBetter(new_model_score)) { + last_update = iter; + models[i].copyTo(new_model); + errors_best = errors; + prev_score = new_model_score; + new_model_score = score; } } - - if (!model_updated) - // if model was not updated at the first iteration then return false - // otherwise if all-inliers LSQ has not updated model then no sense - // to do it again -> return true (model was updated before). - return lsq_iter > 0; - - // if number of inliers doesn't increase more than 5% then break - if (fabs(static_cast(out_score.inlier_number) - static_cast - (best_model_score.inlier_number)) / best_model_score.inlier_number < 0.05) - return true; - - if (lsq_iter != lsq_iterations - 1) - // if not the last LSQ normalization then get inliers for next normalization - inlier_number = quality->getInliers(new_model, inliers); + if (weights.empty()) { + if (iter > last_update) + break; + else { + Quality::getInliers(errors_best, mask, threshold); + if (Utils::intersectionOverUnion(mask, mask_best) >= iou_thr) + return true; + mask_best = mask; + num_inliers = 0; + if (!is_covariance) + for (int p = 0; p < points_size; p++) + if (mask_best[p]) inliers[num_inliers++] = p; + } + } else { + if (iter > last_update) { + // new model is worse + if (CHANGE_WEIGHTS) { + // if failed more than 5 times then break + if (iter - std::max(0, last_update) >= 5) + break; + // try to change weights by changing threshold + if (fabs(new_model_score.score - prev_score.score) < FLT_EPSILON) { + // increase threshold if new model score is the same as the same as previous + max_thr *= 1.05; // increase by 5% + } else if (iter > last_update) { + // decrease max threshold if model is worse + max_thr *= 0.9; // decrease by 10% + } + } else break; // break if not changing weights + } + // generate new weights and inliers + num_inliers = weight_fnc->getInliersWeights(errors_best, inliers, weights, max_thr); + } } - return true; + return last_update >= 0; } }; -Ptr LeastSquaresPolishing::create (const Ptr &estimator_, - const Ptr &quality_, int lsq_iterations_) { - return makePtr(estimator_, quality_, lsq_iterations_); +Ptr NonMinimalPolisher::create(const Ptr &quality_, const Ptr &solver_, + Ptr weight_fnc_, int max_iters_, double iou_thr_) { + return makePtr(quality_, solver_, weight_fnc_, max_iters_, iou_thr_); } }} diff --git a/modules/calib3d/src/usac/pnp_solver.cpp b/modules/calib3d/src/usac/pnp_solver.cpp index 37c649144c..704632a0e8 100644 --- a/modules/calib3d/src/usac/pnp_solver.cpp +++ b/modules/calib3d/src/usac/pnp_solver.cpp @@ -23,7 +23,7 @@ public: int getMaxNumberOfSolutions () const override { return 1; } explicit PnPMinimalSolver6PtsImpl (const Mat &points_) : - points_mat(&points_), points ((float*)points_.data) {} + points_mat(&points_), points ((float*)points_mat->data) {} /* DLT: d x = P X, x = (u, v, 1), X = (X, Y, Z, 1), P = K[R t] @@ -73,6 +73,9 @@ public: int estimate (const std::vector &sample, std::vector &models) const override { std::vector A1 (60, 0), A2(56, 0); // 5x12, 7x8 + // std::vector A_all(11*12, 0); + // int cnt3 = 0; + int cnt1 = 0, cnt2 = 0; for (int i = 0; i < 6; i++) { const int smpl = 5 * sample[i]; @@ -100,14 +103,16 @@ public: A2[cnt2++] = -v * Z; A2[cnt2++] = -v; } - // matrix is sparse -> do not test for singularity + Math::eliminateUpperTriangular(A1, 5, 12); + int offset = 4*12; // add last eliminated row of A1 for (int i = 0; i < 8; i++) A2[cnt2++] = A1[offset + i + 4/* skip 4 first cols*/]; + // must be full-rank if (!Math::eliminateUpperTriangular(A2, 7, 8)) return 0; @@ -139,13 +144,9 @@ public: if (std::isnan(p[i])) return 0; } - models = std::vector{P}; return 1; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; Ptr PnPMinimalSolver6Pts::create(const Mat &points_) { return makePtr(points_); @@ -157,7 +158,7 @@ private: const float * const points; public: explicit PnPNonMinimalSolverImpl (const Mat &points_) : - points_mat(&points_), points ((float*)points_.data){} + points_mat(&points_), points ((float*)points_mat->data){} int estimate (const std::vector &sample, int sample_size, std::vector &models, const std::vector &weights) const override { @@ -236,7 +237,7 @@ public: models = std::vector{ Mat_(3,4) }; Eigen::HouseholderQR> qr((Eigen::Matrix(AtA))); const Eigen::Matrix &Q = qr.householderQ(); - // extract the last nullspace + // extract the last null-vector Eigen::Map>((double *)models[0].data) = Q.col(11); #else Matx Vt; @@ -246,17 +247,37 @@ public: #endif return 1; } - + int estimate (const std::vector &/*mask*/, std::vector &/*models*/, + const std::vector &/*weights*/) override { + return 0; + } + void enforceRankConstraint (bool /*enforce*/) override {} int getMinimumRequiredSampleSize() const override { return 6; } int getMaxNumberOfSolutions () const override { return 1; } - Ptr clone () const override { - return makePtr(*points_mat); - } }; Ptr PnPNonMinimalSolver::create(const Mat &points) { return makePtr(points); } +class PnPSVDSolverImpl : public PnPSVDSolver { +private: + std::vector w; + Ptr pnp_solver; +public: + int getSampleSize() const override { return 6; } + int getMaxNumberOfSolutions () const override { return 1; } + + explicit PnPSVDSolverImpl (const Mat &points_) { + pnp_solver = PnPNonMinimalSolver::create(points_); + } + int estimate (const std::vector &sample, std::vector &models) const override { + return pnp_solver->estimate(sample, 6, models, w); + } +}; +Ptr PnPSVDSolver::create(const Mat &points_) { + return makePtr(points_); +} + class P3PSolverImpl : public P3PSolver { private: /* @@ -264,7 +285,7 @@ private: * K^-1 [u v 1]^T / ||K^-1 [u v 1]^T|| */ const Mat * points_mat, * calib_norm_points_mat; - const Matx33d * K_mat, &K; + const Matx33d K; const float * const calib_norm_points, * const points; const double VAL_THR = 1e-4; public: @@ -272,9 +293,9 @@ public: * @points_ is matrix N x 5 * u v x y z. (u,v) is image point, (x y z) is world point */ - P3PSolverImpl (const Mat &points_, const Mat &calib_norm_points_, const Matx33d &K_) : - points_mat(&points_), calib_norm_points_mat(&calib_norm_points_), K_mat (&K_), - K(K_), calib_norm_points((float*)calib_norm_points_.data), points((float*)points_.data) {} + P3PSolverImpl (const Mat &points_, const Mat &calib_norm_points_, const Mat &K_) : + points_mat(&points_), calib_norm_points_mat(&calib_norm_points_), K(K_), + calib_norm_points((float*)calib_norm_points_mat->data), points((float*)points_mat->data) {} int estimate (const std::vector &sample, std::vector &models) const override { /* @@ -362,8 +383,8 @@ public: zw[2] = Z3crZ1w(0); zw[5] = Z3crZ1w(1); zw[8] = Z3crZ1w(2); const Matx33d R = Math::rotVec2RotMat(Math::rotMat2RotVec(Z * Zw.inv())); - - Mat P, KR = Mat(K * R); + Matx33d KR = K * R; + Matx34d P; hconcat(KR, -KR * (X1 - R.t() * nX1), P); models.emplace_back(P); } @@ -371,11 +392,8 @@ public: } int getSampleSize() const override { return 3; } int getMaxNumberOfSolutions () const override { return 4; } - Ptr clone () const override { - return makePtr(*points_mat, *calib_norm_points_mat, *K_mat); - } }; -Ptr P3PSolver::create(const Mat &points_, const Mat &calib_norm_pts, const Matx33d &K) { +Ptr P3PSolver::create(const Mat &points_, const Mat &calib_norm_pts, const Mat &K) { return makePtr(points_, calib_norm_pts, K); } }} \ No newline at end of file diff --git a/modules/calib3d/src/usac/quality.cpp b/modules/calib3d/src/usac/quality.cpp index d07c1da29d..89c5760c1d 100644 --- a/modules/calib3d/src/usac/quality.cpp +++ b/modules/calib3d/src/usac/quality.cpp @@ -25,6 +25,27 @@ int Quality::getInliers(const Ptr &error, const Mat &model, std::vector &errors, std::vector &inliers, double threshold) { + std::fill(inliers.begin(), inliers.end(), false); + int cnt = 0, inls = 0; + for (const auto e : errors) { + if (e < threshold) { + inliers[cnt] = true; + inls++; + } + cnt++; + } + return inls; +} +int Quality::getInliers (const std::vector &errors, std::vector &inliers, double threshold) { + int cnt = 0, inls = 0; + for (const auto e : errors) { + if (e < threshold) + inliers[inls++] = cnt; + cnt++; + } + return inls; +} class RansacQualityImpl : public RansacQuality { private: @@ -41,14 +62,23 @@ public: Score getScore (const Mat &model) const override { error->setModelParameters(model); int inlier_number = 0; - for (int point = 0; point < points_size; point++) { + const auto preemptive_thr = -points_size - best_score; + for (int point = 0; point < points_size; point++) if (error->getError(point) < threshold) inlier_number++; - if (inlier_number + (points_size - point) < -best_score) - break; - } + else if (inlier_number - point < preemptive_thr) + break; // score is negative inlier number! If less then better - return Score(inlier_number, -static_cast(inlier_number)); + return {inlier_number, -static_cast(inlier_number)}; + } + + Score getScore (const std::vector &errors) const override { + int inlier_number = 0; + for (int point = 0; point < points_size; point++) + if (errors[point] < threshold) + inlier_number++; + // score is negative inlier number! If less then better + return {inlier_number, -static_cast(inlier_number)}; } void setBestScore(double best_score_) override { @@ -61,11 +91,9 @@ public: { return Quality::getInliers(error, model, inliers, thr); } int getInliers (const Mat &model, std::vector &inliers_mask) const override { return Quality::getInliers(error, model, inliers_mask, threshold); } - + double getThreshold () const override { return threshold; } int getPointsSize () const override { return points_size; } - Ptr clone () const override { - return makePtr(points_size, threshold, error->clone()); - } + Ptr getErrorFnc () const override { return error; } }; Ptr RansacQuality::create(int points_size_, double threshold_, @@ -77,13 +105,13 @@ class MsacQualityImpl : public MsacQuality { protected: const Ptr error; const int points_size; - const double threshold; + const double threshold, k_msac; double best_score, norm_thr, one_over_thr; public: - MsacQualityImpl (int points_size_, double threshold_, const Ptr &error_) - : error (error_), points_size (points_size_), threshold (threshold_) { + MsacQualityImpl (int points_size_, double threshold_, const Ptr &error_, double k_msac_) + : error (error_), points_size (points_size_), threshold (threshold_), k_msac(k_msac_) { best_score = std::numeric_limits::max(); - norm_thr = threshold*9/4; + norm_thr = threshold*k_msac; one_over_thr = 1 / norm_thr; } @@ -91,17 +119,31 @@ public: error->setModelParameters(model); double err, sum_errors = 0; int inlier_number = 0; + const auto preemptive_thr = points_size + best_score; for (int point = 0; point < points_size; point++) { err = error->getError(point); if (err < norm_thr) { sum_errors -= (1 - err * one_over_thr); if (err < threshold) inlier_number++; - } - if (sum_errors - points_size + point > best_score) + } else if (sum_errors + point > preemptive_thr) break; } - return Score(inlier_number, sum_errors); + return {inlier_number, sum_errors}; + } + + Score getScore (const std::vector &errors) const override { + double sum_errors = 0; + int inlier_number = 0; + for (int point = 0; point < points_size; point++) { + const auto err = errors[point]; + if (err < norm_thr) { + sum_errors -= (1 - err * one_over_thr); + if (err < threshold) + inlier_number++; + } + } + return {inlier_number, sum_errors}; } void setBestScore(double best_score_) override { @@ -114,103 +156,61 @@ public: { return Quality::getInliers(error, model, inliers, thr); } int getInliers (const Mat &model, std::vector &inliers_mask) const override { return Quality::getInliers(error, model, inliers_mask, threshold); } - + double getThreshold () const override { return threshold; } int getPointsSize () const override { return points_size; } - Ptr clone () const override { - return makePtr(points_size, threshold, error->clone()); - } + Ptr getErrorFnc () const override { return error; } }; Ptr MsacQuality::create(int points_size_, double threshold_, - const Ptr &error_) { - return makePtr(points_size_, threshold_, error_); + const Ptr &error_, double k_msac) { + return makePtr(points_size_, threshold_, error_, k_msac); } class MagsacQualityImpl : public MagsacQuality { private: const Ptr error; - const GammaValues& gamma_generator; + const Ptr gamma_generator; const int points_size; - // for example, maximum standard deviation of noise. const double maximum_threshold_sqr, tentative_inlier_threshold; - // The degrees of freedom of the data from which the model is estimated. - // E.g., for models coming from point correspondences (x1,y1,x2,y2), it is 4. - const int degrees_of_freedom; - // A 0.99 quantile of the Chi^2-distribution to convert sigma values to residuals - const double k; - // Calculating k^2 / 2 which will be used for the estimation and, - // due to being constant, it is better to calculate it a priori. - double squared_k_per_2; - // Calculating (DoF - 1) / 2 which will be used for the estimation and, - // due to being constant, it is better to calculate it a priori. - double dof_minus_one_per_two; - // Calculating (DoF + 1) / 2 which will be used for the estimation and, - // due to being constant, it is better to calculate it a priori. - double dof_plus_one_per_two; - const double C; - // Calculating 2^(DoF - 1) which will be used for the estimation and, - // due to being constant, it is better to calculate it a priori. - double two_ad_dof_minus_one; - // Calculating 2^(DoF + 1) which will be used for the estimation and, - // due to being constant, it is better to calculate it a priori. - double two_ad_dof_plus_one; // Calculate the gamma value of k const double gamma_value_of_k; - // Calculate the lower incomplete gamma value of k - const double lower_gamma_value_of_k; double previous_best_loss; - // Convert the maximum threshold to a sigma value - float maximum_sigma; - // Calculate the squared maximum sigma - float maximum_sigma_2; - // Calculate \sigma_{max}^2 / 2 float maximum_sigma_2_per_2; - // Calculate 2 * \sigma_{max}^2 - float maximum_sigma_2_times_2; // Calculating 2^(DoF + 1) / \sigma_{max} which will be used for the estimation and, // due to being constant, it is better to calculate it a priori. - double two_ad_dof_plus_one_per_maximum_sigma; - double scale_of_stored_incomplete_gammas; - double max_loss; + double two_ad_dof_plus_one_per_maximum_sigma, rescale_err, norm_loss; const std::vector &stored_complete_gamma_values, &stored_lower_incomplete_gamma_values; - int stored_incomplete_gamma_number_min1; + unsigned int stored_incomplete_gamma_number_min1; public: MagsacQualityImpl (double maximum_thr, int points_size_, const Ptr &error_, + const Ptr &gamma_generator_, double tentative_inlier_threshold_, int DoF, double sigma_quantile, - double upper_incomplete_of_sigma_quantile, - double lower_incomplete_of_sigma_quantile, double C_) - : error (error_), gamma_generator(GammaValues::getSingleton()), points_size(points_size_), + double upper_incomplete_of_sigma_quantile) + : error (error_), gamma_generator(gamma_generator_), points_size(points_size_), maximum_threshold_sqr(maximum_thr*maximum_thr), - tentative_inlier_threshold(tentative_inlier_threshold_), degrees_of_freedom(DoF), - k(sigma_quantile), C(C_), gamma_value_of_k (upper_incomplete_of_sigma_quantile), - lower_gamma_value_of_k (lower_incomplete_of_sigma_quantile), - stored_complete_gamma_values(gamma_generator.getCompleteGammaValues()), - stored_lower_incomplete_gamma_values(gamma_generator.getIncompleteGammaValues()) - { + tentative_inlier_threshold(tentative_inlier_threshold_), + gamma_value_of_k (upper_incomplete_of_sigma_quantile), + stored_complete_gamma_values (gamma_generator->getCompleteGammaValues()), + stored_lower_incomplete_gamma_values (gamma_generator->getIncompleteGammaValues()) { previous_best_loss = std::numeric_limits::max(); - squared_k_per_2 = k * k / 2.0; - dof_minus_one_per_two = (degrees_of_freedom - 1.0) / 2.0; - dof_plus_one_per_two = (degrees_of_freedom + 1.0) / 2.0; - two_ad_dof_minus_one = std::pow(2.0, dof_minus_one_per_two); - two_ad_dof_plus_one = std::pow(2.0, dof_plus_one_per_two); - maximum_sigma = (float)sqrt(maximum_threshold_sqr) / (float) k; - maximum_sigma_2 = maximum_sigma * maximum_sigma; + const auto maximum_sigma = (float)sqrt(maximum_threshold_sqr) / sigma_quantile; + const auto maximum_sigma_2 = (float) (maximum_sigma * maximum_sigma); maximum_sigma_2_per_2 = maximum_sigma_2 / 2.f; - maximum_sigma_2_times_2 = maximum_sigma_2 * 2.f; - two_ad_dof_plus_one_per_maximum_sigma = two_ad_dof_plus_one / maximum_sigma; - scale_of_stored_incomplete_gammas = gamma_generator.getScaleOfGammaCompleteValues(); - stored_incomplete_gamma_number_min1 = gamma_generator.getTableSize()-1; - max_loss = 1e-10; - // MAGSAC maximum / minimum loss does not have to be in extrumum residuals - // make 50 iterations to find maximum loss + const auto maximum_sigma_2_times_2 = maximum_sigma_2 * 2.f; + two_ad_dof_plus_one_per_maximum_sigma = pow(2.0, (DoF + 1.0)*.5)/maximum_sigma; + rescale_err = gamma_generator->getScaleOfGammaCompleteValues() / maximum_sigma_2_times_2; + stored_incomplete_gamma_number_min1 = static_cast(gamma_generator->getTableSize()-1); + + double max_loss = 1e-10; + // MAGSAC maximum / minimum loss does not have to be in extremum residuals + // make 30 iterations to find maximum loss const double step = maximum_threshold_sqr / 30; double sqr_res = 0; while (sqr_res < maximum_threshold_sqr) { - int x=(int)round(scale_of_stored_incomplete_gammas * sqr_res - / maximum_sigma_2_times_2); - if (x >= stored_incomplete_gamma_number_min1 || x < 0 /*overflow*/) - x = stored_incomplete_gamma_number_min1; + auto x= static_cast(rescale_err * sqr_res); + if (x > stored_incomplete_gamma_number_min1) + x = stored_incomplete_gamma_number_min1; const double loss = two_ad_dof_plus_one_per_maximum_sigma * (maximum_sigma_2_per_2 * stored_lower_incomplete_gamma_values[x] + sqr_res * 0.25 * (stored_complete_gamma_values[x] - gamma_value_of_k)); @@ -218,6 +218,7 @@ public: max_loss = loss; sqr_res += step; } + norm_loss = two_ad_dof_plus_one_per_maximum_sigma / max_loss; } // https://github.com/danini/magsac @@ -225,26 +226,25 @@ public: error->setModelParameters(model); double total_loss = 0.0; int num_tentative_inliers = 0; + const auto preemptive_thr = points_size + previous_best_loss; for (int point_idx = 0; point_idx < points_size; point_idx++) { const float squared_residual = error->getError(point_idx); if (squared_residual < tentative_inlier_threshold) num_tentative_inliers++; if (squared_residual < maximum_threshold_sqr) { // consider point as inlier // Get the position of the gamma value in the lookup table - int x=(int)round(scale_of_stored_incomplete_gammas * squared_residual - / maximum_sigma_2_times_2); + auto x = static_cast(rescale_err * squared_residual); // If the sought gamma value is not stored in the lookup, return the closest element - if (x >= stored_incomplete_gamma_number_min1 || x < 0 /*overflow*/) - x = stored_incomplete_gamma_number_min1; + if (x > stored_incomplete_gamma_number_min1) + x = stored_incomplete_gamma_number_min1; // Calculate the loss implied by the current point - total_loss -= (1 - two_ad_dof_plus_one_per_maximum_sigma * (maximum_sigma_2_per_2 * + total_loss -= (1 - (maximum_sigma_2_per_2 * stored_lower_incomplete_gamma_values[x] + squared_residual * 0.25 * - (stored_complete_gamma_values[x] - gamma_value_of_k)) / max_loss); - } - if (total_loss - (points_size - point_idx) > previous_best_loss) + (stored_complete_gamma_values[x] - gamma_value_of_k)) * norm_loss); + } else if (total_loss + point_idx > preemptive_thr) break; } - return Score(num_tentative_inliers, total_loss); + return {num_tentative_inliers, total_loss}; } Score getScore (const std::vector &errors) const override { @@ -255,18 +255,15 @@ public: if (squared_residual < tentative_inlier_threshold) num_tentative_inliers++; if (squared_residual < maximum_threshold_sqr) { - int x=(int)round(scale_of_stored_incomplete_gammas * squared_residual - / maximum_sigma_2_times_2); - if (x >= stored_incomplete_gamma_number_min1 || x < 0 /*overflow*/) - x = stored_incomplete_gamma_number_min1; - total_loss -= (1 - two_ad_dof_plus_one_per_maximum_sigma * (maximum_sigma_2_per_2 * + auto x = static_cast(rescale_err * squared_residual); + if (x > stored_incomplete_gamma_number_min1) + x = stored_incomplete_gamma_number_min1; + total_loss -= (1 - (maximum_sigma_2_per_2 * stored_lower_incomplete_gamma_values[x] + squared_residual * 0.25 * - (stored_complete_gamma_values[x] - gamma_value_of_k)) / max_loss); + (stored_complete_gamma_values[x] - gamma_value_of_k)) * norm_loss); } - if (total_loss - (points_size - point_idx) > previous_best_loss) - break; } - return Score(num_tentative_inliers, total_loss); + return {num_tentative_inliers, total_loss}; } void setBestScore (double best_loss) override { @@ -279,20 +276,16 @@ public: { return Quality::getInliers(error, model, inliers, thr); } int getInliers (const Mat &model, std::vector &inliers_mask) const override { return Quality::getInliers(error, model, inliers_mask, tentative_inlier_threshold); } + double getThreshold () const override { return tentative_inlier_threshold; } int getPointsSize () const override { return points_size; } - Ptr clone () const override { - return makePtr(maximum_sigma, points_size, error->clone(), - tentative_inlier_threshold, degrees_of_freedom, - k, gamma_value_of_k, lower_gamma_value_of_k, C); - } + Ptr getErrorFnc () const override { return error; } }; Ptr MagsacQuality::create(double maximum_thr, int points_size_, const Ptr &error_, + const Ptr &gamma_generator, double tentative_inlier_threshold_, int DoF, double sigma_quantile, - double upper_incomplete_of_sigma_quantile, - double lower_incomplete_of_sigma_quantile, double C_) { - return makePtr(maximum_thr, points_size_, error_, - tentative_inlier_threshold_, DoF, sigma_quantile, upper_incomplete_of_sigma_quantile, - lower_incomplete_of_sigma_quantile, C_); + double upper_incomplete_of_sigma_quantile) { + return makePtr(maximum_thr, points_size_, error_, gamma_generator, + tentative_inlier_threshold_, DoF, sigma_quantile, upper_incomplete_of_sigma_quantile); } class LMedsQualityImpl : public LMedsQuality { @@ -312,7 +305,16 @@ public: if (errors[point] < threshold) inlier_number++; // score is median of errors - return Score(inlier_number, Utils::findMedian (errors)); + return {inlier_number, Utils::findMedian (errors)}; + } + Score getScore (const std::vector &errors_) const override { + std::vector errors = errors_; + int inlier_number = 0; + for (int point = 0; point < points_size; point++) + if (errors[point] < threshold) + inlier_number++; + // score is median of errors + return {inlier_number, Utils::findMedian (errors)}; } void setBestScore (double /*best_score*/) override {} @@ -324,10 +326,8 @@ public: { return Quality::getInliers(error, model, inliers, thr); } int getInliers (const Mat &model, std::vector &inliers_mask) const override { return Quality::getInliers(error, model, inliers_mask, threshold); } - - Ptr clone () const override { - return makePtr(points_size, threshold, error->clone()); - } + double getThreshold () const override { return threshold; } + Ptr getErrorFnc () const override { return error; } }; Ptr LMedsQuality::create(int points_size_, double threshold_, const Ptr &error_) { return makePtr(points_size_, threshold_, error_); @@ -335,47 +335,53 @@ Ptr LMedsQuality::create(int points_size_, double threshold_, cons class ModelVerifierImpl : public ModelVerifier { private: - std::vector errors; + Ptr quality; public: - inline bool isModelGood(const Mat &/*model*/) override { return true; } - inline bool getScore(Score &/*score*/) const override { return false; } - void update (int /*highest_inlier_number*/) override {} - const std::vector &getErrors() const override { return errors; } - bool hasErrors () const override { return false; } - Ptr clone (int /*state*/) const override { return makePtr();} + ModelVerifierImpl (const Ptr &q) : quality(q) {} + inline bool isModelGood(const Mat &model, Score &score) override { + score = quality->getScore(model); + return true; + } + void update (const Score &/*score*/, int /*iteration*/) override {} + void reset() override {} + void updateSPRT (double , double , double , double , double , const Score &) override {} }; -Ptr ModelVerifier::create() { - return makePtr(); +Ptr ModelVerifier::create(const Ptr &quality) { + return makePtr(quality); } -///////////////////////////////////// SPRT VERIFIER ////////////////////////////////////////// -class SPRTImpl : public SPRT { +class AdaptiveSPRTImpl : public AdaptiveSPRT { private: RNG rng; const Ptr err; + const Ptr quality; const int points_size; - int highest_inlier_number, current_sprt_idx; // i + int highest_inlier_number, last_iteration; // time t_M needed to instantiate a model hypothesis given a sample // Let m_S be the number of models that are verified per sample - const double inlier_threshold, norm_thr, one_over_thr, t_M, m_S; + const double inlier_threshold, norm_thr, one_over_thr; - double lowest_sum_errors, current_epsilon, current_delta, current_A, - delta_to_epsilon, complement_delta_to_complement_epsilon; + // alpha is false negative rate, alpha = 1 / A + double t_M, lowest_sum_errors, current_epsilon, current_delta, current_A, + delta_to_epsilon, complement_delta_to_complement_epsilon, + time_ver_corr_sprt = 0, time_ver_corr = 0, + one_over_complement_alpha, avg_num_checked_pts; - std::vector sprt_histories; + std::vector sprt_histories, empty; std::vector points_random_pool; std::vector errors; - Score score; + bool do_sprt, adapt, IS_ADAPTIVE; const ScoreMethod score_type; - bool last_model_is_good, can_compute_score, has_errors; + double m_S; public: - SPRTImpl (int state, const Ptr &err_, int points_size_, - double inlier_threshold_, double prob_pt_of_good_model, double prob_pt_of_bad_model, - double time_sample, double avg_num_models, ScoreMethod score_type_) : rng(state), err(err_), - points_size(points_size_), inlier_threshold (inlier_threshold_), - norm_thr(inlier_threshold_*9/4), one_over_thr (1/norm_thr), t_M (time_sample), - m_S (avg_num_models), score_type (score_type_) { + AdaptiveSPRTImpl (int state, const Ptr &quality_, int points_size_, + double inlier_threshold_, double prob_pt_of_good_model, double prob_pt_of_bad_model, + double time_sample, double avg_num_models, ScoreMethod score_type_, + double k_mlesac_, bool is_adaptive) : rng(state), err(quality_->getErrorFnc()), + quality(quality_), points_size(points_size_), inlier_threshold (quality->getThreshold()), + norm_thr(inlier_threshold_*k_mlesac_), one_over_thr (1/norm_thr), t_M (time_sample), + score_type (score_type_), m_S (avg_num_models) { // Generate array of random points for randomized evaluation points_random_pool = std::vector (points_size_); @@ -387,19 +393,23 @@ public: // reserve (approximately) some space for sprt vector. sprt_histories.reserve(20); - createTest(prob_pt_of_good_model, prob_pt_of_bad_model); - - highest_inlier_number = 0; + highest_inlier_number = last_iteration = 0; lowest_sum_errors = std::numeric_limits::max(); - last_model_is_good = false; - can_compute_score = score_type_ == ScoreMethod::SCORE_METHOD_MSAC - || score_type_ == ScoreMethod::SCORE_METHOD_RANSAC - || score_type_ == ScoreMethod::SCORE_METHOD_LMEDS; - // for MSAC and RANSAC errors not needed - if (score_type_ != ScoreMethod::SCORE_METHOD_MSAC && score_type_ != ScoreMethod::SCORE_METHOD_RANSAC) - errors = std::vector(points_size_); - // however return errors only if we can't compute score - has_errors = !can_compute_score; + if (score_type_ != ScoreMethod::SCORE_METHOD_MSAC) + errors = std::vector(points_size_); + IS_ADAPTIVE = is_adaptive; + delta_to_epsilon = one_over_complement_alpha = complement_delta_to_complement_epsilon = current_A = -1; + avg_num_checked_pts = points_size_; + adapt = IS_ADAPTIVE; + do_sprt = !IS_ADAPTIVE; + if (IS_ADAPTIVE) { + // all these variables will be initialized later + current_epsilon = prob_pt_of_good_model; + current_delta = prob_pt_of_bad_model; + } else { + current_epsilon = current_delta = 1e-5; + createTest(prob_pt_of_good_model, prob_pt_of_bad_model); + } } /* @@ -421,153 +431,143 @@ public: * @current_hypothesis: current RANSAC iteration * Return: true if model is good, false - otherwise. */ - inline bool isModelGood(const Mat& model) override - { - if (model.empty()) - return false; - + inline bool isModelGood (const Mat &model, Score &out_score) override { // update error object with current model - err->setModelParameters(model); - - double lambda = 1, sum_errors = 0; - last_model_is_good = true; - int random_pool_idx = rng.uniform(0, points_size), tested_point, tested_inliers = 0; - for (tested_point = 0; tested_point < points_size; tested_point++) { - if (random_pool_idx >= points_size) - random_pool_idx = 0; - const double error = err->getError (points_random_pool[random_pool_idx++]); - if (error < inlier_threshold) { - tested_inliers++; - lambda *= delta_to_epsilon; - } else { - lambda *= complement_delta_to_complement_epsilon; - // since delta is always higher than epsilon, then lambda can increase only - // when point is not consistent with model - if (lambda > current_A) - break; - } + bool last_model_is_good = true; + double sum_errors = 0; + int tested_inliers = 0; + if (! do_sprt || adapt) { // if adapt or not sprt then compute model score directly + out_score = quality->getScore(model); + tested_inliers = out_score.inlier_number; + sum_errors = out_score.score; + } else { // do sprt and not adapt + err->setModelParameters(model); + double lambda = 1; + int random_pool_idx = rng.uniform(0, points_size), tested_point; if (score_type == ScoreMethod::SCORE_METHOD_MSAC) { - if (error < norm_thr) - sum_errors -= (1 - error * one_over_thr); - if (sum_errors - points_size + tested_point > lowest_sum_errors) - break; - } else if (score_type == ScoreMethod::SCORE_METHOD_RANSAC) { - if (tested_inliers + points_size - tested_point < highest_inlier_number) - break; - } else errors[points_random_pool[random_pool_idx-1]] = (float)error; + const auto preemptive_thr = points_size + lowest_sum_errors; + for (tested_point = 0; tested_point < points_size; tested_point++) { + if (random_pool_idx == points_size) + random_pool_idx = 0; + const float error = err->getError (points_random_pool[random_pool_idx++]); + if (error < inlier_threshold) { + tested_inliers++; + lambda *= delta_to_epsilon; + } else { + lambda *= complement_delta_to_complement_epsilon; + // since delta is always higher than epsilon, then lambda can increase only + // when point is not consistent with model + if (lambda > current_A) + break; + } + if (error < norm_thr) + sum_errors -= (1 - error * one_over_thr); + else if (sum_errors + tested_point > preemptive_thr) + break; + } + } else { // save errors into array here + for (tested_point = 0; tested_point < points_size; tested_point++) { + if (random_pool_idx == points_size) + random_pool_idx = 0; + const int pt = points_random_pool[random_pool_idx++]; + const float error = err->getError (pt); + if (error < inlier_threshold) { + tested_inliers++; + lambda *= delta_to_epsilon; + } else { + lambda *= complement_delta_to_complement_epsilon; + if (lambda > current_A) + break; + } + errors[pt] = error; + } + } + last_model_is_good = tested_point == points_size; } - last_model_is_good = tested_point == points_size; - - // increase number of samples processed by current test - sprt_histories[current_sprt_idx].tested_samples++; - if (last_model_is_good) { - score.inlier_number = tested_inliers; - if (score_type == ScoreMethod::SCORE_METHOD_MSAC) { - score.score = sum_errors; - if (lowest_sum_errors > sum_errors) - lowest_sum_errors = sum_errors; - } else if (score_type == ScoreMethod::SCORE_METHOD_RANSAC) - score.score = -static_cast(tested_inliers); - else if (score_type == ScoreMethod::SCORE_METHOD_LMEDS) - score.score = Utils::findMedian(errors); - - const double new_epsilon = static_cast(tested_inliers) / points_size; - if (new_epsilon > current_epsilon) { - highest_inlier_number = tested_inliers; // update max inlier number - /* - * Model accepted and the largest support so far: - * design (i+1)-th test (εi + 1= εˆ, δi+1 = δ, i := i + 1). - * Store the current model parameters θ - */ - createTest(new_epsilon, current_delta); - } - } else { - /* - * Since almost all tested models are ‘bad’, the probability - * δ can be estimated as the average fraction of consistent data points - * in rejected models. - */ - // add 1 to tested_point, because loop over tested_point starts from 0 - const double delta_estimated = static_cast (tested_inliers) / (tested_point+1); - if (delta_estimated > 0 && fabs(current_delta - delta_estimated) - / current_delta > 0.05) - /* - * Model rejected: re-estimate δ. If the estimate δ_ differs - * from δi by more than 5% design (i+1)-th test (εi+1 = εi, - * δi+1 = δˆ, i := i + 1) - */ - createTest(current_epsilon, delta_estimated); + if (last_model_is_good && do_sprt) { + out_score.inlier_number = tested_inliers; + if (score_type == ScoreMethod::SCORE_METHOD_MSAC) + out_score.score = sum_errors; + else if (score_type == ScoreMethod::SCORE_METHOD_RANSAC) + out_score.score = -static_cast(tested_inliers); + else out_score = quality->getScore(errors); } return last_model_is_good; } - inline bool getScore (Score &score_) const override { - if (!last_model_is_good || !can_compute_score) - return false; - score_ = score; - return true; - } - bool hasErrors () const override { return has_errors; } - const std::vector &getErrors () const override { return errors; } - const std::vector &getSPRTvector () const override { return sprt_histories; } - void update (int highest_inlier_number_) override { - const double new_epsilon = static_cast(highest_inlier_number_) / points_size; - if (new_epsilon > current_epsilon) { - highest_inlier_number = highest_inlier_number_; - if (sprt_histories[current_sprt_idx].tested_samples == 0) - sprt_histories[current_sprt_idx].tested_samples = 1; - // save sprt test and create new one - createTest(new_epsilon, current_delta); + // update SPRT parameters = called only once inside usac + void updateSPRT (double time_model_est, double time_corr_ver, double new_avg_models, double new_delta, double new_epsilon, const Score &best_score) override { + if (adapt) { + adapt = false; + m_S = new_avg_models; + t_M = time_model_est / time_corr_ver; + time_ver_corr = time_corr_ver; + time_ver_corr_sprt = time_corr_ver * 1.05; + createTest(new_epsilon, new_delta); + highest_inlier_number = best_score.inlier_number; + lowest_sum_errors = best_score.score; } } - Ptr clone (int state) const override { - return makePtr(state, err->clone(), points_size, inlier_threshold, - sprt_histories[current_sprt_idx].epsilon, - sprt_histories[current_sprt_idx].delta, t_M, m_S, score_type); + + const std::vector &getSPRTvector () const override { return adapt ? empty : sprt_histories; } + void update (const Score &score, int iteration) override { + if (adapt || highest_inlier_number > score.inlier_number) + return; + + if (sprt_histories.size() == 1 && sprt_histories.back().tested_samples == 0) + sprt_histories.back().tested_samples = iteration; + else if (! sprt_histories.empty()) + sprt_histories.back().tested_samples += iteration - last_iteration; + + SPRT_history new_sprt_history; + new_sprt_history.epsilon = (double)score.inlier_number / points_size; + highest_inlier_number = score.inlier_number; + lowest_sum_errors = score.score; + createTest(static_cast(highest_inlier_number) / points_size, current_delta); + new_sprt_history.delta = current_delta; + new_sprt_history.A = current_A; + sprt_histories.emplace_back(new_sprt_history); + last_iteration = iteration; + } + int avgNumCheckedPts () const override { return do_sprt ? (int)avg_num_checked_pts + 1 : points_size; } + void reset() override { + adapt = true; + do_sprt = false; + highest_inlier_number = last_iteration = 0; + lowest_sum_errors = DBL_MAX; + sprt_histories.clear(); } private: - - // Saves sprt test to sprt history and update current epsilon, delta and threshold. - void createTest (double epsilon, double delta) { + // Update current epsilon, delta and threshold (A). + bool createTest (double epsilon, double delta) { + if (fabs(current_epsilon - epsilon) < FLT_EPSILON && fabs(current_delta - delta) < FLT_EPSILON) + return false; // if epsilon is closed to 1 then set them to 0.99 to avoid numerical problems if (epsilon > 0.999999) epsilon = 0.999; // delta can't be higher than epsilon, because ratio delta / epsilon will be greater than 1 - if (epsilon < delta) delta = epsilon-0.0001; + if (epsilon < delta) delta = epsilon-0.001; // avoid delta going too high as it is very unlikely // e.g., 30% of points are consistent with bad model is not very real if (delta > 0.3) delta = 0.3; - SPRT_history new_sprt_history; - new_sprt_history.epsilon = epsilon; - new_sprt_history.delta = delta; - new_sprt_history.A = estimateThresholdA (epsilon, delta); - - sprt_histories.emplace_back(new_sprt_history); - - current_A = new_sprt_history.A; + const auto AC = estimateThresholdA (epsilon, delta); + current_A = AC.first; + const auto C = AC.second; current_delta = delta; current_epsilon = epsilon; + one_over_complement_alpha = 1 / (1 - 1 / current_A); delta_to_epsilon = delta / epsilon; complement_delta_to_complement_epsilon = (1 - delta) / (1 - epsilon); - current_sprt_idx = static_cast(sprt_histories.size()) - 1; - } - /* - * A(0) = K1/K2 + 1 - * A(n+1) = K1/K2 + 1 + log (A(n)) - * K1 = t_M / P_g - * K2 = m_S/(P_g*C) - * t_M is time needed to instantiate a model hypotheses given a sample - * P_g = epsilon ^ m, m is the number of data point in the Ransac sample. - * m_S is the number of models that are verified per sample. - * p (0|Hb) p (1|Hb) - * C = p(0|Hb) log (---------) + p(1|Hb) log (---------) - * p (0|Hg) p (1|Hg) - */ - double estimateThresholdA (double epsilon, double delta) { - const double C = (1 - delta) * log ((1 - delta) / (1 - epsilon)) + - delta * (log(delta / epsilon)); + if (IS_ADAPTIVE) { + avg_num_checked_pts = std::min((log(current_A) / C) * one_over_complement_alpha, (double)points_size); + do_sprt = time_ver_corr_sprt * avg_num_checked_pts < time_ver_corr * points_size; + } + return true; + } + std::pair estimateThresholdA (double epsilon, double delta) { + const double C = (1 - delta) * log ((1 - delta) / (1 - epsilon)) + delta * log (delta / epsilon); // K = K1/K2 + 1 = (t_M / P_g) / (m_S / (C * P_g)) + 1 = (t_M * C)/m_S + 1 const double K = t_M * C / m_S + 1; double An, An_1 = K; @@ -579,13 +579,13 @@ private: break; An_1 = An; } - return An; + return std::make_pair(An, C); } }; -Ptr SPRT::create (int state, const Ptr &err_, int points_size_, - double inlier_threshold_, double prob_pt_of_good_model, double prob_pt_of_bad_model, - double time_sample, double avg_num_models, ScoreMethod score_type_) { - return makePtr(state, err_, points_size_, inlier_threshold_, - prob_pt_of_good_model, prob_pt_of_bad_model, time_sample, avg_num_models, score_type_); +Ptr AdaptiveSPRT::create (int state, const Ptr &quality, int points_size_, + double inlier_threshold_, double prob_pt_of_good_model, double prob_pt_of_bad_model, + double time_sample, double avg_num_models, ScoreMethod score_type_, double k_mlesac, bool is_adaptive) { + return makePtr(state, quality, points_size_, inlier_threshold_, + prob_pt_of_good_model, prob_pt_of_bad_model, time_sample, avg_num_models, score_type_, k_mlesac, is_adaptive); } }} diff --git a/modules/calib3d/src/usac/ransac_solvers.cpp b/modules/calib3d/src/usac/ransac_solvers.cpp index cca13405bc..1a76353331 100644 --- a/modules/calib3d/src/usac/ransac_solvers.cpp +++ b/modules/calib3d/src/usac/ransac_solvers.cpp @@ -6,51 +6,54 @@ #include "../usac.hpp" #include -namespace cv { namespace usac { +namespace cv { +UsacParams::UsacParams() { + confidence=0.99; + isParallel=false; + loIterations=5; + loMethod=LOCAL_OPTIM_INNER_LO; + loSampleSize=14; + maxIterations=5000; + neighborsSearch=NEIGH_GRID; + randomGeneratorState=0; + sampler=SAMPLING_UNIFORM; + score=SCORE_METHOD_MSAC; + threshold=1.5; + final_polisher=COV_POLISHER; + final_polisher_iterations=3; +} + +namespace usac { int mergePoints (InputArray pts1_, InputArray pts2_, Mat &pts, bool ispnp); void setParameters (int flag, Ptr ¶ms, EstimationMethod estimator, double thr, int max_iters, double conf, bool mask_needed); class RansacOutputImpl : public RansacOutput { private: - Mat model; - // vector of number_inliers size std::vector inliers; - // vector of points size, true if inlier, false-outlier + cv::Mat model, K1, K2; + // vector of number_inliers size + // vector of points size, true if inlier, false - outlier std::vector inliers_mask; // vector of points size, value of i-th index corresponds to error of i-th point if i is inlier. - std::vector errors; - // the best found score of RANSAC - double score; - - int seconds, milliseconds, microseconds; - int time_mcs, number_inliers, number_estimated_models, number_good_models; - int number_iterations; // number of iterations of main RANSAC + std::vector residuals; + int number_inliers, number_iterations; + ModelConfidence conf; public: - RansacOutputImpl (const Mat &model_, const std::vector &inliers_mask_, - int time_mcs_, double score_, int number_inliers_, int number_iterations_, - int number_estimated_models_, int number_good_models_) { - + RansacOutputImpl (const cv::Mat &model_, const std::vector &inliers_mask_, int number_inliers_, + int number_iterations_, ModelConfidence conf_, const std::vector &errors_) { model_.copyTo(model); inliers_mask = inliers_mask_; - time_mcs = time_mcs_; - score = score_; number_inliers = number_inliers_; number_iterations = number_iterations_; - number_estimated_models = number_estimated_models_; - number_good_models = number_good_models_; - microseconds = time_mcs % 1000; - milliseconds = ((time_mcs - microseconds)/1000) % 1000; - seconds = ((time_mcs - 1000*milliseconds - microseconds)/(1000*1000)) % 60; + residuals = errors_; + conf = conf_; } - /* - * Return inliers' indices. - * size of vector = number of inliers - */ + // Return inliers' indices of size = number of inliers const std::vector &getInliers() override { if (inliers.empty()) { - inliers.reserve(inliers_mask.size()); + inliers.reserve(number_inliers); int pt_cnt = 0; for (bool is_inlier : inliers_mask) { if (is_inlier) @@ -60,319 +63,968 @@ public: } return inliers; } - - // Return inliers mask. Vector of points size. 1-inlier, 0-outlier. - const std::vector &getInliersMask() const override { return inliers_mask; } - - int getTimeMicroSeconds() const override {return time_mcs; } - int getTimeMicroSeconds1() const override {return microseconds; } - int getTimeMilliSeconds2() const override {return milliseconds; } - int getTimeSeconds3() const override {return seconds; } - int getNumberOfInliers() const override { return number_inliers; } - int getNumberOfMainIterations() const override { return number_iterations; } - int getNumberOfGoodModels () const override { return number_good_models; } - int getNumberOfEstimatedModels () const override { return number_estimated_models; } - const Mat &getModel() const override { return model; } + const std::vector &getInliersMask() const override { + return inliers_mask; + } + int getNumberOfInliers() const override { + return number_inliers; + } + const Mat &getModel() const override { + return model; + } + int getNumberOfIters() const override { + return number_iterations; + } + ModelConfidence getConfidence() const override { + return conf; + } + const std::vector &getResiduals() const override { + return residuals; + } }; -Ptr RansacOutput::create(const Mat &model_, const std::vector &inliers_mask_, - int time_mcs_, double score_, int number_inliers_, int number_iterations_, - int number_estimated_models_, int number_good_models_) { - return makePtr(model_, inliers_mask_, time_mcs_, score_, number_inliers_, - number_iterations_, number_estimated_models_, number_good_models_); +Ptr RansacOutput::create(const cv::Mat &model_, const std::vector &inliers_mask_, int number_inliers_, + int number_iterations_, ModelConfidence conf, const std::vector &errors_) { + return makePtr(model_, inliers_mask_, number_inliers_, + number_iterations_, conf, errors_); +} + +double getLambda (std::vector &supports, double cdf_thr, int points_size, + int sample_size, bool is_independent, int &min_non_random_inliers) { + std::sort(supports.begin(), supports.end()); + double lambda = supports.size() % 2 ? (supports[supports.size()/2] + supports[supports.size()/2+1])*0.5 : supports[supports.size()/2]; + const double cdf = lambda + cdf_thr*sqrt(lambda * (1 - lambda / (is_independent ? points_size - sample_size : points_size))); + int lower_than_cdf = 0; lambda = 0; + for (const auto &inl : supports) + if (inl < cdf) { + lambda += inl; lower_than_cdf++; + } else break; // list is sorted + lambda /= lower_than_cdf; + if (lambda < 1 || lower_than_cdf == 0) lambda = 1; + // use 0.9999 quantile https://keisan.casio.com/exec/system/14060745333941 + if (! is_independent) // do not calculate it for all inliers + min_non_random_inliers = (int)(lambda + 2.32*sqrt(lambda * (1 - lambda / points_size))) + 1; + return lambda; } class Ransac { -protected: - const Ptr params; - const Ptr _estimator; - const Ptr _quality; - const Ptr _sampler; - const Ptr _termination_criteria; - const Ptr _model_verifier; - const Ptr _degeneracy; - const Ptr _local_optimization; - const Ptr model_polisher; - - const int points_size, state; - const bool parallel; public: + const Ptr params; + Ptr _estimator; + Ptr _error; + Ptr _quality; + Ptr _sampler; + Ptr _termination; + Ptr _model_verifier; + Ptr _degeneracy; + Ptr _local_optimization; + Ptr polisher; + Ptr _gamma_generator; + Ptr _min_solver; + Ptr _lo_solver, _fo_solver; + Ptr _lo_sampler; + Ptr _weight_fnc; - Ransac (const Ptr ¶ms_, int points_size_, const Ptr &estimator_, const Ptr &quality_, - const Ptr &sampler_, const Ptr &termination_criteria_, - const Ptr &model_verifier_, const Ptr °eneracy_, - const Ptr &local_optimization_, const Ptr &model_polisher_, - bool parallel_=false, int state_ = 0) : + int points_size, _state, filtered_points_size; + double threshold, max_thr; + bool parallel; - params (params_), _estimator (estimator_), _quality (quality_), _sampler (sampler_), - _termination_criteria (termination_criteria_), _model_verifier (model_verifier_), - _degeneracy (degeneracy_), _local_optimization (local_optimization_), - model_polisher (model_polisher_), points_size (points_size_), state(state_), - parallel(parallel_) {} + Matx33d T1, T2; + Mat points, K1, K2, calib_points, image_points, norm_points, filtered_points; + Ptr graph; + std::vector> layers; + + Ransac (const Ptr ¶ms_, cv::InputArray points1, cv::InputArray points2, + cv::InputArray K1_, cv::InputArray K2_, cv::InputArray dist_coeff1, cv::InputArray dist_coeff2) : params(params_) { + _state = params->getRandomGeneratorState(); + threshold = params->getThreshold(); + max_thr = std::max(threshold, params->getMaximumThreshold()); + parallel = params->isParallel(); + Mat undist_points1, undist_points2; + if (params->isPnP()) { + if (! K1_.empty()) { + K1 = K1_.getMat().clone(); K1.convertTo(K1, CV_64F); + if (! dist_coeff1.empty()) { + // undistortPoints also calibrate points using K + undistortPoints(points1.isContinuous() ? points1 : points1.getMat().clone(), undist_points1, K1_, dist_coeff1); + points_size = mergePoints(undist_points1, points2, points, true); + Utils::normalizeAndDecalibPointsPnP (K1, points, calib_points); + } else { + points_size = mergePoints(points1, points2, points, true); + Utils::calibrateAndNormalizePointsPnP(K1, points, calib_points); + } + } else points_size = mergePoints(points1, points2, points, true); + } else { + if (params->isEssential()) { + CV_CheckEQ((int)(!K1_.empty() && !K2_.empty()), 1, "Intrinsic matrix must not be empty!"); + K1 = K1_.getMat(); K1.convertTo(K1, CV_64F); + K2 = K2_.getMat(); K2.convertTo(K2, CV_64F); + if (! dist_coeff1.empty() || ! dist_coeff2.empty()) { + // undistortPoints also calibrate points using K + if (! dist_coeff1.empty()) undistortPoints(points1.isContinuous() ? points1 : points1.getMat().clone(), undist_points1, K1_, dist_coeff1); + else undist_points1 = points1.getMat(); + if (! dist_coeff2.empty()) undistortPoints(points2.isContinuous() ? points2 : points2.getMat().clone(), undist_points2, K2_, dist_coeff2); + else undist_points2 = points2.getMat(); + points_size = mergePoints(undist_points1, undist_points2, calib_points, false); + } else { + points_size = mergePoints(points1, points2, points, false); + Utils::calibratePoints(K1, K2, points, calib_points); + } + threshold = Utils::getCalibratedThreshold(threshold, K1, K2); + max_thr = Utils::getCalibratedThreshold(max_thr, K1, K2); + } else { + points_size = mergePoints(points1, points2, points, false); + if (params->isFundamental() && ! K1_.empty() && ! K2_.empty()) { + K1 = K1_.getMat(); K1.convertTo(K1, CV_64F); + K2 = K2_.getMat(); K2.convertTo(K2, CV_64F); + Utils::calibratePoints(K1, K2, points, calib_points); + } + } + } + + if (params->getSampler() == SamplingMethod::SAMPLING_NAPSAC || params->getLO() == LocalOptimMethod::LOCAL_OPTIM_GC) { + if (params->getNeighborsSearch() == NeighborSearchMethod::NEIGH_GRID) { + graph = GridNeighborhoodGraph::create(points, points_size, + params->getCellSize(), params->getCellSize(), params->getCellSize(), params->getCellSize(), 10); + } else if (params->getNeighborsSearch() == NeighborSearchMethod::NEIGH_FLANN_KNN) { + graph = FlannNeighborhoodGraph::create(points, points_size,params->getKNN(), false, 5, 1); + } else if (params->getNeighborsSearch() == NeighborSearchMethod::NEIGH_FLANN_RADIUS) { + graph = RadiusSearchNeighborhoodGraph::create(points, points_size, params->getGraphRadius(), 5, 1); + } else CV_Error(cv::Error::StsNotImplemented, "Graph type is not implemented!"); + } + + if (params->getSampler() == SamplingMethod::SAMPLING_PROGRESSIVE_NAPSAC) { + CV_CheckEQ((int)params->isPnP(), 0, "ProgressiveNAPSAC for PnP is not implemented!"); + const auto &cell_number_per_layer = params->getGridCellNumber(); + layers.reserve(cell_number_per_layer.size()); + const auto * const pts = (float *) points.data; + float img1_width = 0, img1_height = 0, img2_width = 0, img2_height = 0; + for (int i = 0; i < 4 * points_size; i += 4) { + if (pts[i ] > img1_width ) img1_width = pts[i ]; + if (pts[i + 1] > img1_height) img1_height = pts[i + 1]; + if (pts[i + 2] > img2_width ) img2_width = pts[i + 2]; + if (pts[i + 3] > img2_height) img2_height = pts[i + 3]; + } + // Create grid graphs (overlapping layes of given cell numbers) + for (int layer_idx = 0; layer_idx < (int)cell_number_per_layer.size(); layer_idx++) { + const int cell_number = cell_number_per_layer[layer_idx]; + if (layer_idx > 0) + if (cell_number_per_layer[layer_idx-1] <= cell_number) + CV_Error(cv::Error::StsError, "Progressive NAPSAC sampler: " + "Cell number in layers must be in decreasing order!"); + layers.emplace_back(GridNeighborhoodGraph::create(points, points_size, + (int)(img1_width / (float)cell_number), (int)(img1_height / (float)cell_number), + (int)(img2_width / (float)cell_number), (int)(img2_height / (float)cell_number), 10)); + } + } + + // update points by calibrated for Essential matrix after graph is calculated + if (params->isEssential()) { + image_points = points; + points = calib_points; + // if maximum calibrated threshold significanlty differs threshold then set upper bound + if (max_thr > 10*threshold) + max_thr = 10*threshold; + } + + // Since error function output squared error distance, so make + // threshold squared as well + threshold *= threshold; + + if ((params->isHomography() || (params->isFundamental() && (K1.empty() || K2.empty() || !params->isLarssonOptimization())) || + params->getEstimator() == EstimationMethod::AFFINE) && (params->getLO() != LOCAL_OPTIM_NULL || params->getFinalPolisher() == COV_POLISHER)) { + const auto normTr = NormTransform::create(points); + std::vector sample (points_size); + for (int i = 0; i < points_size; i++) sample[i] = i; + normTr->getNormTransformation(norm_points, sample, points_size, T1, T2); + } + + if (params->getScore() == SCORE_METHOD_MAGSAC || params->getLO() == LOCAL_OPTIM_SIGMA || params->getFinalPolisher() == MAGSAC) + _gamma_generator = GammaValues::create(params->getDegreesOfFreedom()); // is thread safe + initialize (_state, _min_solver, _lo_solver, _error, _estimator, _degeneracy, _quality, + _model_verifier, _local_optimization, _termination, _sampler, _lo_sampler, _weight_fnc, false/*parallel*/); + if (params->getFinalPolisher() != NONE_POLISHER) { + polisher = NonMinimalPolisher::create(_quality, _fo_solver, + params->getFinalPolisher() == MAGSAC ? _weight_fnc : nullptr, params->getFinalLSQIterations(), 0.99); + } + } + + void initialize (int state, Ptr &min_solver, Ptr &non_min_solver, + Ptr &error, Ptr &estimator, Ptr °eneracy, Ptr &quality, + Ptr &verifier, Ptr &lo, Ptr &termination, + Ptr &sampler, Ptr &lo_sampler, Ptr &weight_fnc, bool parallel_call) { + + const int min_sample_size = params->getSampleSize(), prosac_termination_length = std::min((int)(.5*points_size), 100); + // inner inlier threshold will be used in LO to obtain inliers + // additionally in DEGENSAC for F + double inner_inlier_thr_sqr = threshold; + if (params->isHomography() && inner_inlier_thr_sqr < 5.25) inner_inlier_thr_sqr = 5.25; // at least 2.5 px + else if (params->isFundamental() && inner_inlier_thr_sqr < 4) inner_inlier_thr_sqr = 4; // at least 2 px + + if (params->getFinalPolisher() == MAGSAC || params->getLO() == LOCAL_OPTIM_SIGMA) + weight_fnc = MagsacWeightFunction::create(_gamma_generator, params->getDegreesOfFreedom(), params->getUpperIncompleteOfSigmaQuantile(), params->getC(), params->getMaximumThreshold()); + else weight_fnc = nullptr; + + switch (params->getError()) { + case ErrorMetric::SYMM_REPR_ERR: + error = ReprojectionErrorSymmetric::create(points); break; + case ErrorMetric::FORW_REPR_ERR: + if (params->getEstimator() == EstimationMethod::AFFINE) + error = ReprojectionErrorAffine::create(points); + else error = ReprojectionErrorForward::create(points); + break; + case ErrorMetric::SAMPSON_ERR: + error = SampsonError::create(points); break; + case ErrorMetric::SGD_ERR: + error = SymmetricGeometricDistance::create(points); break; + case ErrorMetric::RERPOJ: + error = ReprojectionErrorPmatrix::create(points); break; + default: CV_Error(cv::Error::StsNotImplemented , "Error metric is not implemented!"); + } + + const double k_mlesac = params->getKmlesac (); + switch (params->getScore()) { + case ScoreMethod::SCORE_METHOD_RANSAC : + quality = RansacQuality::create(points_size, threshold, error); break; + case ScoreMethod::SCORE_METHOD_MSAC : + quality = MsacQuality::create(points_size, threshold, error, k_mlesac); break; + case ScoreMethod::SCORE_METHOD_MAGSAC : + quality = MagsacQuality::create(max_thr, points_size, error, _gamma_generator, + threshold, params->getDegreesOfFreedom(), params->getSigmaQuantile(), + params->getUpperIncompleteOfSigmaQuantile()); break; + case ScoreMethod::SCORE_METHOD_LMEDS : + quality = LMedsQuality::create(points_size, threshold, error); break; + default: CV_Error(cv::Error::StsNotImplemented, "Score is not imeplemeted!"); + } + + const auto is_ge_solver = params->getRansacSolver() == GEM_SOLVER; + if (params->isHomography()) { + degeneracy = HomographyDegeneracy::create(points); + min_solver = HomographyMinimalSolver4pts::create(points, is_ge_solver); + non_min_solver = HomographyNonMinimalSolver::create(norm_points, T1, T2, true); + estimator = HomographyEstimator::create(min_solver, non_min_solver, degeneracy); + if (!parallel_call && params->getFinalPolisher() != NONE_POLISHER) { + if (params->getFinalPolisher() == COV_POLISHER) + _fo_solver = CovarianceHomographySolver::create(norm_points, T1, T2); + else _fo_solver = HomographyNonMinimalSolver::create(points); + } + } else if (params->isFundamental()) { + if (K1.empty() || K2.empty()) { + degeneracy = FundamentalDegeneracy::create(state++, quality, points, min_sample_size, + params->getPlaneAndParallaxIters(), std::max(threshold, 8.) /*sqr homogr thr*/, inner_inlier_thr_sqr, K1, K2); + } else degeneracy = FundamentalDegeneracyViaE::create(quality, points, calib_points, K1, K2, true/*is F*/); + if (min_sample_size == 7) { + min_solver = FundamentalMinimalSolver7pts::create(points, is_ge_solver); + } else min_solver = FundamentalMinimalSolver8pts::create(points); + if (params->isLarssonOptimization() && !K1.empty() && !K2.empty()) { + non_min_solver = LarssonOptimizer::create(calib_points, K1, K2, params->getLevMarqItersLO(), true/*F*/); + } else { + if (weight_fnc) + non_min_solver = EpipolarNonMinimalSolver::create(points, true); + else + non_min_solver = EpipolarNonMinimalSolver::create(norm_points, T1, T2, true); + } + estimator = FundamentalEstimator::create(min_solver, non_min_solver, degeneracy); + if (!parallel_call && params->getFinalPolisher() != NONE_POLISHER) { + if (params->isLarssonOptimization() && !K1.empty() && !K2.empty()) + _fo_solver = LarssonOptimizer::create(calib_points, K1, K2, params->getLevMarqIters(), true/*F*/); + else if (params->getFinalPolisher() == COV_POLISHER) + _fo_solver = CovarianceEpipolarSolver::create(norm_points, T1, T2); + else _fo_solver = EpipolarNonMinimalSolver::create(points, true); + } + } else if (params->isEssential()) { + if (params->getEstimator() == EstimationMethod::ESSENTIAL) { + min_solver = EssentialMinimalSolver5pts::create(points, !is_ge_solver, true/*Nister*/); + degeneracy = EssentialDegeneracy::create(points, min_sample_size); + } + non_min_solver = LarssonOptimizer::create(calib_points, K1, K2, params->getLevMarqItersLO(), false/*E*/); + estimator = EssentialEstimator::create(min_solver, non_min_solver, degeneracy); + if (!parallel_call && params->getFinalPolisher() != NONE_POLISHER) + _fo_solver = LarssonOptimizer::create(calib_points, K1, K2, params->getLevMarqIters(), false/*E*/); + } else if (params->isPnP()) { + degeneracy = makePtr(); + if (min_sample_size == 3) { + min_solver = P3PSolver::create(points, calib_points, K1); + non_min_solver = DLSPnP::create(points, calib_points, K1); + } else { + if (is_ge_solver) + min_solver = PnPMinimalSolver6Pts::create(points); + else min_solver = PnPSVDSolver::create(points); + non_min_solver = PnPNonMinimalSolver::create(points); + } + estimator = PnPEstimator::create(min_solver, non_min_solver); + if (!parallel_call && params->getFinalPolisher() != NONE_POLISHER) _fo_solver = non_min_solver; + } else if (params->getEstimator() == EstimationMethod::AFFINE) { + degeneracy = makePtr(); + min_solver = AffineMinimalSolver::create(points); + non_min_solver = AffineNonMinimalSolver::create(points, cv::noArray(), cv::noArray()); + estimator = AffineEstimator::create(min_solver, non_min_solver); + if (!parallel_call && params->getFinalPolisher() != NONE_POLISHER) { + if (params->getFinalPolisher() == COV_POLISHER) + _fo_solver = CovarianceAffineSolver::create(points); + else _fo_solver = non_min_solver; + } + } else CV_Error(cv::Error::StsNotImplemented, "Estimator not implemented!"); + + switch (params->getSampler()) { + case SamplingMethod::SAMPLING_UNIFORM: + sampler = UniformSampler::create(state++, min_sample_size, points_size); + break; + case SamplingMethod::SAMPLING_PROSAC: + if (!parallel_call) // for parallel only one PROSAC sampler + sampler = ProsacSampler::create(state++, points_size, min_sample_size, params->getProsacMaxSamples()); + break; + case SamplingMethod::SAMPLING_PROGRESSIVE_NAPSAC: + sampler = ProgressiveNapsac::create(state++, points_size, min_sample_size, layers, 20); break; + case SamplingMethod::SAMPLING_NAPSAC: + sampler = NapsacSampler::create(state++, points_size, min_sample_size, graph); break; + default: CV_Error(cv::Error::StsNotImplemented, "Sampler is not implemented!"); + } + + const bool is_sprt = params->getVerifier() == VerificationMethod::SPRT_VERIFIER || params->getVerifier() == VerificationMethod::ASPRT; + if (is_sprt) + verifier = AdaptiveSPRT::create(state++, quality, points_size, params->getScore() == ScoreMethod ::SCORE_METHOD_MAGSAC ? max_thr : threshold, + params->getSPRTepsilon(), params->getSPRTdelta(), params->getTimeForModelEstimation(), + params->getSPRTavgNumModels(), params->getScore(), k_mlesac, params->getVerifier() == VerificationMethod::ASPRT); + else if (params->getVerifier() == VerificationMethod::NULL_VERIFIER) + verifier = ModelVerifier::create(quality); + else CV_Error(cv::Error::StsNotImplemented, "Verifier is not imeplemented!"); + + if (params->getSampler() == SamplingMethod::SAMPLING_PROSAC) { + if (parallel_call) { + termination = ProsacTerminationCriteria::create(nullptr, error, + points_size, min_sample_size, params->getConfidence(), params->getMaxIters(), prosac_termination_length, 0.05, 0.05, threshold, + _termination.dynamicCast()->getNonRandomInliers()); + } else { + termination = ProsacTerminationCriteria::create(sampler.dynamicCast(), error, + points_size, min_sample_size, params->getConfidence(), params->getMaxIters(), prosac_termination_length, 0.05, 0.05, threshold, + std::vector()); + } + } else if (params->getSampler() == SamplingMethod::SAMPLING_PROGRESSIVE_NAPSAC) { + if (is_sprt) + termination = SPRTPNapsacTermination::create(verifier.dynamicCast(), + params->getConfidence(), points_size, min_sample_size, + params->getMaxIters(), params->getRelaxCoef()); + else termination = StandardTerminationCriteria::create (params->getConfidence(), + points_size, min_sample_size, params->getMaxIters()); + } else if (is_sprt && params->getLO() == LocalOptimMethod::LOCAL_OPTIM_NULL) { + termination = SPRTTermination::create(verifier.dynamicCast(), + params->getConfidence(), points_size, min_sample_size, params->getMaxIters()); + } else { + termination = StandardTerminationCriteria::create + (params->getConfidence(), points_size, min_sample_size, params->getMaxIters()); + } + + // if normal ransac or parallel call, avoid redundant init + if ((! params->isParallel() || parallel_call) && params->getLO() != LocalOptimMethod::LOCAL_OPTIM_NULL) { + lo_sampler = UniformRandomGenerator::create(state, points_size, params->getLOSampleSize()); + const auto lo_termination = StandardTerminationCriteria::create(params->getConfidence(), points_size, min_sample_size, params->getMaxIters()); + switch (params->getLO()) { + case LocalOptimMethod::LOCAL_OPTIM_INNER_LO: case LocalOptimMethod::LOCAL_OPTIM_SIGMA: + lo = SimpleLocalOptimization::create(quality, non_min_solver, lo_termination, lo_sampler, + weight_fnc, params->getLOInnerMaxIters(), inner_inlier_thr_sqr, true); break; + case LocalOptimMethod::LOCAL_OPTIM_INNER_AND_ITER_LO: + lo = InnerIterativeLocalOptimization::create(estimator, quality, lo_sampler, + points_size, threshold, true, params->getLOIterativeSampleSize(), + params->getLOInnerMaxIters(), params->getLOIterativeMaxIters(), + params->getLOThresholdMultiplier()); break; + case LocalOptimMethod::LOCAL_OPTIM_GC: + lo = GraphCut::create(estimator, quality, graph, lo_sampler, threshold, + params->getGraphCutSpatialCoherenceTerm(), params->getLOInnerMaxIters(), lo_termination); break; + default: CV_Error(cv::Error::StsNotImplemented , "Local Optimization is not implemented!"); + } + } + } + + int getIndependentInliers (const Mat &model_, const std::vector &sample, + std::vector &inliers, const int num_inliers_) { + bool is_F = params->isFundamental(); + Mat model = model_; + int sample_size = 0; + if (is_F) sample_size = 7; + else if (params->isHomography()) sample_size = 4; + else if (params->isEssential()) { + is_F = true; + // convert E to F + model = Mat(Matx33d(K2).inv().t() * Matx33d(model) * Matx33d(K1).inv()); + sample_size = 5; + } else if (params->isPnP() || params->getEstimator() == EstimationMethod::AFFINE) sample_size = 3; + else + CV_Error(cv::Error::StsNotImplemented, "Method for independent inliers is not implemented for this problem"); + if (num_inliers_ <= sample_size) return 0; // minimal sample size generates model + model.convertTo(model, CV_32F); + int num_inliers = num_inliers_, num_pts_near_ep = 0, + num_pts_validatin_or_constr = 0, pt1 = 0; + const auto * const pts = params->isEssential() ? (float *) image_points.data : (float *) points.data; + // scale for thresholds should be used + const float ep_thr_sqr = 0.000001f, line_thr = 0.01f, neigh_thr = 4.0f; + float sign1=0,a1=0, b1=0, c1=0, a2=0, b2=0, c2=0, ep1_x, ep1_y, ep2_x, ep2_y; + const auto * const m = (float *) model.data; + Vec3f ep1; + bool do_or_test = false, ep1_inf = false, ep2_inf = false; + if (is_F) { // compute epipole and sign of the first point for orientation test + model *= (1/norm(model)); + ep1 = Utils::getRightEpipole(model); + const Vec3f ep2 = Utils::getLeftEpipole(model); + if (fabsf(ep1[2]) < DBL_EPSILON) { + ep1_inf = true; + } else { + ep1_x = ep1[0] / ep1[2]; + ep1_y = ep1[1] / ep1[2]; + } + if (fabsf(ep2[2]) < DBL_EPSILON) { + ep2_inf = true; + } else { + ep2_x = ep2[0] / ep2[2]; + ep2_y = ep2[1] / ep2[2]; + } + } + const auto * const e1 = ep1.val; // of size 3x1 + + // we move sample points to the end, so every inlier will be checked by sample point + int num_sample_in_inliers = 0; + if (!sample.empty()) { + num_sample_in_inliers = 0; + int temp_idx = num_inliers; + for (int i = 0; i < temp_idx; i++) { + const int inl = inliers[i]; + for (int s : sample) { + if (inl == s) { + std::swap(inliers[i], inliers[--temp_idx]); + i--; // we need to check inlier that we just swapped + num_sample_in_inliers++; + break; + } + } + } + } + + if (is_F) { + int MIN_TEST = std::min(15, num_inliers); + for (int i = 0; i < MIN_TEST; i++) { + pt1 = 4*inliers[i]; + sign1 = (m[0]*pts[pt1+2]+m[3]*pts[pt1+3]+m[6])*(e1[1]-e1[2]*pts[pt1+1]); + int validate = 0; + for (int j = 0; j < MIN_TEST; j++) { + if (i == j) continue; + const int inl_idx = 4*inliers[j]; + if (sign1*(m[0]*pts[inl_idx+2]+m[3]*pts[inl_idx+3]+m[6])*(e1[1]-e1[2]*pts[inl_idx+1])<0) + validate++; + } + if (validate < MIN_TEST/2) { + do_or_test = true; break; + } + } + } + + // verification does not include sample points as they are surely random + const int max_verify = num_inliers - num_sample_in_inliers; + if (max_verify <= 0) + return 0; + int num_non_random_inliers = num_inliers - sample_size; + auto removeDependentPoints = [&] (bool do_orient_test, bool check_epipoles) { + for (int i = 0; i < max_verify; i++) { + // checks over inliers if they are dependent to other inliers + const int inl_idx = 4*inliers[i]; + const auto x1 = pts[inl_idx], y1 = pts[inl_idx+1], x2 = pts[inl_idx+2], y2 = pts[inl_idx+3]; + if (is_F) { + // epipolar line on image 2 = l2 + a2 = m[0] * x1 + m[1] * y1 + m[2]; + b2 = m[3] * x1 + m[4] * y1 + m[5]; + c2 = m[6] * x1 + m[7] * y1 + m[8]; + // epipolar line on image 1 = l1 + a1 = m[0] * x2 + m[3] * y2 + m[6]; + b1 = m[1] * x2 + m[4] * y2 + m[7]; + c1 = m[2] * x2 + m[5] * y2 + m[8]; + if ((!ep1_inf && fabsf(x1-ep1_x)+fabsf(y1-ep1_y) < neigh_thr) || + (!ep2_inf && fabsf(x2-ep2_x)+fabsf(y2-ep2_y) < neigh_thr)) { + num_non_random_inliers--; + num_pts_near_ep++; + continue; // is dependent, continue to the next point + } else if (check_epipoles) { + if (a2 * a2 + b2 * b2 + c2 * c2 < ep_thr_sqr || + a1 * a1 + b1 * b1 + c1 * c1 < ep_thr_sqr) { + num_non_random_inliers--; + num_pts_near_ep++; + continue; // is dependent, continue to the next point + } + } + else if (do_orient_test && pt1 != inl_idx && sign1*(m[0]*x2+m[3]*y2+m[6])*(e1[1]-e1[2]*y1)<0) { + num_non_random_inliers--; + num_pts_validatin_or_constr++; + continue; + } + const auto mag2 = 1 / sqrt(a2 * a2 + b2 * b2), mag1 = 1/sqrt(a1 * a1 + b1 * b1); + a2 *= mag2; b2 *= mag2; c2 *= mag2; + a1 *= mag1; b1 *= mag1; c1 *= mag1; + } + + for (int j = i+1; j < num_inliers; j++) {// verify through all including sample points + const int inl_idx_j = 4*inliers[j]; + const auto X1 = pts[inl_idx_j], Y1 = pts[inl_idx_j+1], X2 = pts[inl_idx_j+2], Y2 = pts[inl_idx_j+3]; + // use L1 norm instead of L2 for faster evaluation + if (fabsf(X1-x1) + fabsf(Y1 - y1) < neigh_thr || fabsf(X2-x2) + fabsf(Y2 - y2) < neigh_thr) { + num_non_random_inliers--; + // num_pts_bad_conditioning++; + break; // is dependent stop verification + } else if (is_F) { + if (fabsf(a2 * X2 + b2 * Y2 + c2) < line_thr && //|| // xj'^T F xi + fabsf(a1 * X1 + b1 * Y1 + c1) < line_thr) { // xj^T F^T xi' + num_non_random_inliers--; + break; // is dependent stop verification + } + } + } + } + }; + if (params->isPnP()) { + for (int i = 0; i < max_verify; i++) { + const int inl_idx = 5*inliers[i]; + const auto x = pts[inl_idx], y = pts[inl_idx+1], X = pts[inl_idx+2], Y = pts[inl_idx+3], Z = pts[inl_idx+4]; + for (int j = i+1; j < num_inliers; j++) { + const int inl_idx_j = 5*inliers[j]; + if (fabsf(x-pts[inl_idx_j ]) + fabsf(y-pts[inl_idx_j+1]) < neigh_thr || + fabsf(X-pts[inl_idx_j+2]) + fabsf(Y-pts[inl_idx_j+3]) + fabsf(Z-pts[inl_idx_j+4]) < neigh_thr) { + num_non_random_inliers--; + break; + } + } + } + } else { + removeDependentPoints(do_or_test, !ep1_inf && !ep2_inf); + if (is_F) { + const bool is_pts_vald_constr_normal = (double)num_pts_validatin_or_constr / num_inliers < 0.6; + const bool is_pts_near_ep_normal = (double)num_pts_near_ep / num_inliers < 0.6; + if (!is_pts_near_ep_normal || !is_pts_vald_constr_normal) { + num_non_random_inliers = num_inliers-sample_size; + num_pts_near_ep = 0; num_pts_validatin_or_constr = 0; + removeDependentPoints(is_pts_vald_constr_normal, is_pts_near_ep_normal); + } + } + } + return num_non_random_inliers; + } bool run(Ptr &ransac_output) { if (points_size < params->getSampleSize()) return false; + const bool LO = params->getLO() != LocalOptimMethod::LOCAL_OPTIM_NULL, + IS_FUNDAMENTAL = params->isFundamental(), IS_NON_RAND_TEST = params->isNonRandomnessTest(); + const int MAX_MODELS_ADAPT = 21, MAX_ITERS_ADAPT = MAX_MODELS_ADAPT/*assume at least 1 model from 1 sample*/, + sample_size = params->getSampleSize(); + const double IOU_SIMILARITY_THR = 0.80; + std::vector non_degen_sample, best_sample; - const auto begin_time = std::chrono::steady_clock::now(); + double lambda_non_random_all_inliers = -1; + int final_iters, num_total_tested_models = 0; - // check if LO - const bool LO = params->getLO() != LocalOptimMethod::LOCAL_OPTIM_NULL; - const bool is_magsac = params->getLO() == LocalOptimMethod::LOCAL_OPTIM_SIGMA; - const int max_hyp_test_before_ver = params->getMaxNumHypothesisToTestBeforeRejection(); - const int repeat_magsac = 10, max_iters_before_LO = params->getMaxItersBeforeLO(); - Score best_score; - Mat best_model; - int final_iters; + // non-random + const int MAX_TEST_MODELS_NONRAND = IS_NON_RAND_TEST ? MAX_MODELS_ADAPT : 0; + std::vector models_for_random_test; models_for_random_test.reserve(MAX_TEST_MODELS_NONRAND); + std::vector> samples_for_random_test; samples_for_random_test.reserve(MAX_TEST_MODELS_NONRAND); + bool last_model_from_LO = false; + Mat best_model, best_model_not_from_LO, K1_approx, K2_approx; + Score best_score, best_score_model_not_from_LO; + std::vector best_inliers_mask(points_size); if (! parallel) { - auto update_best = [&] (const Mat &new_model, const Score &new_score) { - best_score = new_score; - // remember best model - new_model.copyTo(best_model); - // update quality and verifier to save evaluation time of a model - _quality->setBestScore(best_score.score); - // update verifier - _model_verifier->update(best_score.inlier_number); - // update upper bound of iterations - return _termination_criteria->update(best_model, best_score.inlier_number); - }; - bool was_LO_run = false; + // adaptive sprt test + double IoU = 0, mean_num_est_models = 0; + bool adapt = IS_NON_RAND_TEST || params->getVerifier() == VerificationMethod ::ASPRT, was_LO_run = false; + int min_non_random_inliers = 30, iters = 0, num_estimations = 0, max_iters = params->getMaxIters(); Mat non_degenerate_model, lo_model; - Score current_score, lo_score, non_denegenerate_model_score; - - // reallocate memory for models + Score current_score, non_degenerate_model_score, best_score_sample; + std::vector model_inliers_mask (points_size); std::vector models(_estimator->getMaxNumSolutions()); + std::vector sample(_estimator->getMinimalSampleSize()), supports; + supports.reserve(3*MAX_MODELS_ADAPT); // store model supports during adaption + auto update_best = [&] (const Mat &new_model, const Score &new_score, bool from_lo=false) { + _quality->getInliers(new_model, model_inliers_mask); + IoU = Utils::intersectionOverUnion(best_inliers_mask, model_inliers_mask); + best_inliers_mask = model_inliers_mask; + + if (!best_model.empty() && (int)models_for_random_test.size() < MAX_TEST_MODELS_NONRAND && IoU < IOU_SIMILARITY_THR && !from_lo) { // use IoU to not save similar models + // save old best model for non-randomness test if necessary + models_for_random_test.emplace_back(best_model.clone()); + samples_for_random_test.emplace_back(best_sample); + } + + // update score, model, inliers and max iterations + best_score = new_score; + new_model.copyTo(best_model); + + if (!from_lo) { + best_sample = sample; + if (IS_FUNDAMENTAL) { // avoid degeneracy after LO run + // save last model not from LO + best_model.copyTo(best_model_not_from_LO); + best_score_model_not_from_LO = best_score; + } + } + + _model_verifier->update(best_score, iters); + max_iters = _termination->update(best_model, best_score.inlier_number); + // max_iters = std::max(max_iters, std::min(10, params->getMaxIters())); + if (!adapt) // update quality and verifier to save evaluation time of a model + _quality->setBestScore(best_score.score); + last_model_from_LO = from_lo; + }; + + auto run_lo = [&] (const Mat &_model, const Score &_score, bool force_lo) { + was_LO_run = true; + _local_optimization->setCurrentRANSACiter(force_lo ? iters : -1); + Score lo_score; + if (_local_optimization->refineModel(_model, _score, lo_model, lo_score) && lo_score.isBetter(best_score)) + update_best(lo_model, lo_score, true); + }; - // allocate memory for sample - std::vector sample(_estimator->getMinimalSampleSize()); - int iters = 0, max_iters = params->getMaxIters(); for (; iters < max_iters; iters++) { _sampler->generateSample(sample); - const int number_of_models = _estimator->estimateModels(sample, models); - + int number_of_models; + if (adapt) { + number_of_models = _estimator->estimateModels(sample, models); + mean_num_est_models += number_of_models; + num_estimations++; + } else { + number_of_models = _estimator->estimateModels(sample, models); + } for (int i = 0; i < number_of_models; i++) { - if (iters < max_hyp_test_before_ver) { + num_total_tested_models++; + if (adapt) { current_score = _quality->getScore(models[i]); - } else { - if (is_magsac && iters % repeat_magsac == 0) { - if (!_local_optimization->refineModel - (models[i], best_score, models[i], current_score)) - continue; - } else if (_model_verifier->isModelGood(models[i])) { - if (!_model_verifier->getScore(current_score)) { - if (_model_verifier->hasErrors()) - current_score = _quality->getScore(_model_verifier->getErrors()); - else current_score = _quality->getScore(models[i]); - } - } else continue; - } - - if (current_score.isBetter(best_score)) { - if (_degeneracy->recoverIfDegenerate(sample, models[i], - non_degenerate_model, non_denegenerate_model_score)) { - // check if best non degenerate model is better than so far the best model - if (non_denegenerate_model_score.isBetter(best_score)) - max_iters = update_best(non_degenerate_model, non_denegenerate_model_score); - else continue; - } else max_iters = update_best(models[i], current_score); - - if (LO && iters >= max_iters_before_LO) { - // do magsac if it wasn't already run - if (is_magsac && iters % repeat_magsac == 0 && iters >= max_hyp_test_before_ver) continue; // magsac has already run - was_LO_run = true; - // update model by Local optimization - if (_local_optimization->refineModel - (best_model, best_score, lo_model, lo_score)) { - if (lo_score.isBetter(best_score)){ - max_iters = update_best(lo_model, lo_score); - } - } + supports.emplace_back(current_score.inlier_number); + if (IS_NON_RAND_TEST && best_score_sample.isBetter(current_score)) { + models_for_random_test.emplace_back(models[i].clone()); + samples_for_random_test.emplace_back(sample); } - if (iters > max_iters) - break; + } else { + if (! _model_verifier->isModelGood(models[i], current_score)) + continue; + } + if (current_score.isBetter(best_score_sample)) { + if (_degeneracy->recoverIfDegenerate(sample, models[i], current_score, + non_degenerate_model, non_degenerate_model_score)) { + // check if best non degenerate model is better than so far the best model + if (non_degenerate_model_score.isBetter(best_score)) { + update_best(non_degenerate_model, non_degenerate_model_score); + best_score_sample = current_score.isBetter(best_score) ? best_score : current_score; + } else continue; + } else { + best_score_sample = current_score; + update_best(models[i], current_score); + } + + if (LO && ((iters < max_iters && best_score.inlier_number > min_non_random_inliers && IoU < IOU_SIMILARITY_THR))) + run_lo(best_model, best_score, false); + } // end of if so far the best score } // end loop of number of models - if (LO && !was_LO_run && iters >= max_iters_before_LO) { - was_LO_run = true; - if (_local_optimization->refineModel(best_model, best_score, lo_model, lo_score)) - if (lo_score.isBetter(best_score)){ - max_iters = update_best(lo_model, lo_score); - } + if (adapt && iters >= MAX_ITERS_ADAPT && num_total_tested_models >= MAX_MODELS_ADAPT) { + adapt = false; + lambda_non_random_all_inliers = getLambda(supports, 2.32, points_size, sample_size, false, min_non_random_inliers); + _model_verifier->updateSPRT(params->getTimeForModelEstimation(), 1.0, mean_num_est_models/num_estimations, lambda_non_random_all_inliers/points_size,(double)std::max(min_non_random_inliers, best_score.inlier_number)/points_size, best_score); } } // end main while loop - final_iters = iters; - } else { - const int MAX_THREADS = getNumThreads(); + if (! was_LO_run && !best_model.empty() && LO) + run_lo(best_model, best_score, true); + } else { // parallel VSAC + const int MAX_THREADS = getNumThreads(), growth_max_samples = params->getProsacMaxSamples(); const bool is_prosac = params->getSampler() == SamplingMethod::SAMPLING_PROSAC; - std::atomic_bool success(false); - std::atomic_int num_hypothesis_tested(0); - std::atomic_int thread_cnt(0); - std::vector best_scores(MAX_THREADS); - std::vector best_models(MAX_THREADS); - - Mutex mutex; // only for prosac - + std::atomic_int num_hypothesis_tested(0), thread_cnt(0), max_number_inliers(0), subset_size, termination_length; + std::atomic best_score_all(std::numeric_limits::max()); + std::vector best_scores(MAX_THREADS), best_scores_not_LO; + std::vector best_models(MAX_THREADS), best_models_not_LO, K1_apx, K2_apx; + std::vector num_tested_models_threads(MAX_THREADS), growth_function, non_random_inliers; + std::vector> tested_models_threads(MAX_THREADS); + std::vector>> tested_samples_threads(MAX_THREADS); + std::vector> best_samples_threads(MAX_THREADS); + std::vector last_model_from_LO_vec; + std::vector lambda_non_random_all_inliers_vec(MAX_THREADS); + if (IS_FUNDAMENTAL) { + last_model_from_LO_vec = std::vector(MAX_THREADS); + best_models_not_LO = std::vector(MAX_THREADS); + best_scores_not_LO = std::vector(MAX_THREADS); + K1_apx = std::vector(MAX_THREADS); + K2_apx = std::vector(MAX_THREADS); + } + if (is_prosac) { + growth_function = _sampler.dynamicCast()->getGrowthFunction(); + subset_size = 2*sample_size; // n, size of the current sampling pool + termination_length = points_size; + } /////////////////////////////////////////////////////////////////////////////////////////////////////// parallel_for_(Range(0, MAX_THREADS), [&](const Range & /*range*/) { if (!success) { // cover all if not success to avoid thread creating new variables const int thread_rng_id = thread_cnt++; - int thread_state = state + 10*thread_rng_id; - - Ptr estimator = _estimator->clone(); - Ptr degeneracy = _degeneracy->clone(thread_state++); - Ptr quality = _quality->clone(); - Ptr model_verifier = _model_verifier->clone(thread_state++); // update verifier - Ptr local_optimization; - if (LO) - local_optimization = _local_optimization->clone(thread_state++); - Ptr termination_criteria = _termination_criteria->clone(); + bool adapt = params->getVerifier() == VerificationMethod ::ASPRT || IS_NON_RAND_TEST; + int thread_state = _state + thread_rng_id, min_non_random_inliers = 0, num_tested_models = 0, + num_estimations = 0, mean_num_est_models = 0, iters, max_iters = params->getMaxIters(); + double IoU = 0, lambda_non_random_all_inliers_thread = -1; + std::vector tested_models_thread; tested_models_thread.reserve(MAX_TEST_MODELS_NONRAND); + std::vector> tested_samples_thread; tested_samples_thread.reserve(MAX_TEST_MODELS_NONRAND); + Ptr random_gen; + if (is_prosac) random_gen = UniformRandomGenerator::create(thread_state); + Ptr error; + Ptr estimator; + Ptr degeneracy; + Ptr quality; + Ptr model_verifier; Ptr sampler; - if (!is_prosac) - sampler = _sampler->clone(thread_state); - - Mat best_model_thread, non_degenerate_model, lo_model; - Score best_score_thread, current_score, non_denegenerate_model_score, lo_score, - best_score_all_threads; - std::vector sample(estimator->getMinimalSampleSize()); + Ptr lo_sampler; + Ptr termination; + Ptr local_optimization; + Ptr min_solver; + Ptr non_min_solver; + Ptr weight_fnc; + initialize (thread_state, min_solver, non_min_solver, error, estimator, degeneracy, quality, + model_verifier, local_optimization, termination, sampler, lo_sampler, weight_fnc, true); + bool is_last_from_LO_thread = false; + Mat best_model_thread, non_degenerate_model, lo_model, best_not_LO_thread; + Score best_score_thread, current_score, non_denegenerate_model_score, lo_score,best_score_all_threads, best_not_LO_score_thread; + std::vector sample(estimator->getMinimalSampleSize()), best_sample_thread, supports; + supports.reserve(3*MAX_MODELS_ADAPT); // store model supports + std::vector best_inliers_mask_local(points_size, false), model_inliers_mask(points_size, false); std::vector models(estimator->getMaxNumSolutions()); - int iters, max_iters = params->getMaxIters(); - auto update_best = [&] (const Score &new_score, const Mat &new_model) { + auto update_best = [&] (const Score &new_score, const Mat &new_model, bool from_LO=false) { + // update best score of all threads + if (max_number_inliers < new_score.inlier_number) max_number_inliers = new_score.inlier_number; + if (best_score_all > new_score.score) best_score_all = new_score.score; + best_score_all_threads = Score(max_number_inliers, best_score_all); + // + quality->getInliers(new_model, model_inliers_mask); + IoU = Utils::intersectionOverUnion(best_inliers_mask_local, model_inliers_mask); + if (!best_model_thread.empty() && (int)tested_models_thread.size() < MAX_TEST_MODELS_NONRAND && IoU < IOU_SIMILARITY_THR) { + tested_models_thread.emplace_back(best_model_thread.clone()); + tested_samples_thread.emplace_back(best_sample_thread); + } + if (!adapt) { // update quality and verifier + quality->setBestScore(best_score_all); + model_verifier->update(best_score_all_threads, iters); + } // copy new score to best score best_score_thread = new_score; - best_scores[thread_rng_id] = best_score_thread; + best_sample_thread = sample; + best_inliers_mask_local = model_inliers_mask; // remember best model new_model.copyTo(best_model_thread); - best_model_thread.copyTo(best_models[thread_rng_id]); - best_score_all_threads = best_score_thread; - // update upper bound of iterations - return termination_criteria->update - (best_model_thread, best_score_thread.inlier_number); - }; + // update upper bound of iterations + if (is_prosac) { + int new_termination_length; + max_iters = termination.dynamicCast()-> + updateTerminationLength(best_model_thread, best_score_thread.inlier_number, new_termination_length); + // update termination length + if (new_termination_length < termination_length) + termination_length = new_termination_length; + } else max_iters = termination->update(best_model_thread, max_number_inliers); + if (IS_FUNDAMENTAL) { + is_last_from_LO_thread = from_LO; + if (!from_LO) { + best_model_thread.copyTo(best_not_LO_thread); + best_not_LO_score_thread = best_score_thread; + } + } + }; bool was_LO_run = false; + auto runLO = [&] (int current_ransac_iters) { + was_LO_run = true; + local_optimization->setCurrentRANSACiter(current_ransac_iters); + if (local_optimization->refineModel(best_model_thread, best_score_thread, lo_model, + lo_score) && lo_score.isBetter(best_score_thread)) + update_best(lo_score, lo_model, true); + }; for (iters = 0; iters < max_iters && !success; iters++) { success = num_hypothesis_tested++ > max_iters; - - if (iters % 10) { + if (iters % 10 && !adapt) { // Synchronize threads. just to speed verification of model. - int best_thread_idx = thread_rng_id; - bool updated = false; - for (int t = 0; t < MAX_THREADS; t++) { - if (best_scores[t].isBetter(best_score_all_threads)) { - best_score_all_threads = best_scores[t]; - updated = true; - best_thread_idx = t; - } - } - if (updated && best_thread_idx != thread_rng_id) { - quality->setBestScore(best_score_all_threads.score); - model_verifier->update(best_score_all_threads.inlier_number); - } + quality->setBestScore(std::min(best_score_thread.score, (double)best_score_all)); + model_verifier->update(best_score_thread.inlier_number > max_number_inliers ? best_score_thread : best_score_all_threads, iters); } if (is_prosac) { - // use global sampler - mutex.lock(); - _sampler->generateSample(sample); - mutex.unlock(); + if (num_hypothesis_tested > growth_max_samples) { + // if PROSAC has not converged to solution then do uniform sampling. + random_gen->generateUniqueRandomSet(sample, sample_size, points_size); + } else { + if (num_hypothesis_tested >= growth_function[subset_size-1] && subset_size < termination_length-MAX_THREADS) { + subset_size++; + if (subset_size >= points_size) subset_size = points_size-1; + } + if (growth_function[subset_size-1] < num_hypothesis_tested) { + // The sample contains m-1 points selected from U_(n-1) at random and u_n + random_gen->generateUniqueRandomSet(sample, sample_size-1, subset_size-1); + sample[sample_size-1] = subset_size-1; + } else + // Select m points from U_n at random. + random_gen->generateUniqueRandomSet(sample, sample_size, subset_size); + } } else sampler->generateSample(sample); // use local sampler const int number_of_models = estimator->estimateModels(sample, models); + if (adapt) { + num_estimations++; mean_num_est_models += number_of_models; + } for (int i = 0; i < number_of_models; i++) { - if (iters < max_hyp_test_before_ver) { + num_tested_models++; + if (adapt) { current_score = quality->getScore(models[i]); - } else { - if (is_magsac && iters % repeat_magsac == 0) { - if (local_optimization && !local_optimization->refineModel - (models[i], best_score_thread, models[i], current_score)) - continue; - } else if (model_verifier->isModelGood(models[i])) { - if (!model_verifier->getScore(current_score)) { - if (model_verifier->hasErrors()) - current_score = quality->getScore(model_verifier->getErrors()); - else current_score = quality->getScore(models[i]); - } - } else continue; - } + supports.emplace_back(current_score.inlier_number); + } else if (! model_verifier->isModelGood(models[i], current_score)) + continue; if (current_score.isBetter(best_score_all_threads)) { - if (degeneracy->recoverIfDegenerate(sample, models[i], - non_degenerate_model, non_denegenerate_model_score)) { + if (degeneracy->recoverIfDegenerate(sample, models[i], current_score, + non_degenerate_model, non_denegenerate_model_score)) { // check if best non degenerate model is better than so far the best model if (non_denegenerate_model_score.isBetter(best_score_thread)) - max_iters = update_best(non_denegenerate_model_score, non_degenerate_model); + update_best(non_denegenerate_model_score, non_degenerate_model); else continue; - } else - max_iters = update_best(current_score, models[i]); - - if (LO && iters >= max_iters_before_LO) { - // do magsac if it wasn't already run - if (is_magsac && iters % repeat_magsac == 0 && iters >= max_hyp_test_before_ver) continue; - was_LO_run = true; - // update model by Local optimizaion - if (local_optimization->refineModel - (best_model_thread, best_score_thread, lo_model, lo_score)) - if (lo_score.isBetter(best_score_thread)) { - max_iters = update_best(lo_score, lo_model); - } - } - if (num_hypothesis_tested > max_iters) { - success = true; break; - } + } else update_best(current_score, models[i]); + if (LO && num_hypothesis_tested < max_iters && IoU < IOU_SIMILARITY_THR && + best_score_thread.inlier_number > min_non_random_inliers) + runLO(iters); } // end of if so far the best score + else if ((int)tested_models_thread.size() < MAX_TEST_MODELS_NONRAND) { + tested_models_thread.emplace_back(models[i].clone()); + tested_samples_thread.emplace_back(sample); + } + if (num_hypothesis_tested > max_iters) { + success = true; break; + } } // end loop of number of models - if (LO && !was_LO_run && iters >= max_iters_before_LO) { - was_LO_run = true; - if (_local_optimization->refineModel(best_model, best_score, lo_model, lo_score)) - if (lo_score.isBetter(best_score)){ - max_iters = update_best(lo_score, lo_model); - } + if (adapt && iters >= MAX_ITERS_ADAPT && num_tested_models >= MAX_MODELS_ADAPT) { + adapt = false; + lambda_non_random_all_inliers_thread = getLambda(supports, 2.32, points_size, sample_size, false, min_non_random_inliers); + model_verifier->updateSPRT(params->getTimeForModelEstimation(), 1, (double)mean_num_est_models/num_estimations, lambda_non_random_all_inliers_thread/points_size, + (double)std::max(min_non_random_inliers, best_score.inlier_number)/points_size, best_score_all_threads); } + if (!adapt && LO && num_hypothesis_tested < max_iters && !was_LO_run && !best_model_thread.empty() && + best_score_thread.inlier_number > min_non_random_inliers) + runLO(iters); } // end of loop over iters + if (! was_LO_run && !best_model_thread.empty() && LO) + runLO(-1 /*use full iterations of LO*/); + best_model_thread.copyTo(best_models[thread_rng_id]); + best_scores[thread_rng_id] = best_score_thread; + num_tested_models_threads[thread_rng_id] = num_tested_models; + tested_models_threads[thread_rng_id] = tested_models_thread; + tested_samples_threads[thread_rng_id] = tested_samples_thread; + best_samples_threads[thread_rng_id] = best_sample_thread; + if (IS_FUNDAMENTAL) { + best_scores_not_LO[thread_rng_id] = best_not_LO_score_thread; + best_not_LO_thread.copyTo(best_models_not_LO[thread_rng_id]); + last_model_from_LO_vec[thread_rng_id] = is_last_from_LO_thread; + } + lambda_non_random_all_inliers_vec[thread_rng_id] = lambda_non_random_all_inliers_thread; }}); // end parallel /////////////////////////////////////////////////////////////////////////////////////////////////////// // find best model from all threads' models best_score = best_scores[0]; int best_thread_idx = 0; - for (int i = 1; i < MAX_THREADS; i++) { + for (int i = 1; i < MAX_THREADS; i++) if (best_scores[i].isBetter(best_score)) { best_score = best_scores[i]; best_thread_idx = i; } - } best_model = best_models[best_thread_idx]; + if (IS_FUNDAMENTAL) { + last_model_from_LO = last_model_from_LO_vec[best_thread_idx]; + K1_approx = K1_apx[best_thread_idx]; + K2_approx = K2_apx[best_thread_idx]; + } final_iters = num_hypothesis_tested; + best_sample = best_samples_threads[best_thread_idx]; + int num_lambdas = 0; + double avg_lambda = 0; + for (int i = 0; i < MAX_THREADS; i++) { + if (IS_FUNDAMENTAL && best_scores_not_LO[i].isBetter(best_score_model_not_from_LO)) { + best_score_model_not_from_LO = best_scores_not_LO[i]; + best_models_not_LO[i].copyTo(best_model_not_from_LO); + } + if (IS_NON_RAND_TEST && lambda_non_random_all_inliers_vec[i] > 0) { + num_lambdas ++; + avg_lambda += lambda_non_random_all_inliers_vec[i]; + } + num_total_tested_models += num_tested_models_threads[i]; + if ((int)models_for_random_test.size() < MAX_TEST_MODELS_NONRAND) { + for (int m = 0; m < (int)tested_models_threads[i].size(); m++) { + models_for_random_test.emplace_back(tested_models_threads[i][m].clone()); + samples_for_random_test.emplace_back(tested_samples_threads[i][m]); + if ((int)models_for_random_test.size() == MAX_TEST_MODELS_NONRAND) + break; + } + } + } + if (IS_NON_RAND_TEST && num_lambdas > 0 && avg_lambda > 0) + lambda_non_random_all_inliers = avg_lambda / num_lambdas; } - - if (best_model.empty()) + if (best_model.empty()) { + ransac_output = RansacOutput::create(best_model, std::vector(), best_score.inlier_number, final_iters, ModelConfidence::RANDOM, std::vector()); return false; - - // polish final model - if (params->getFinalPolisher() != PolishingMethod::NonePolisher) { + } + if (last_model_from_LO && IS_FUNDAMENTAL && K1.empty() && K2.empty()) { + Score new_score; Mat new_model; + const double INL_THR = 0.80; + if (parallel) + _quality->getInliers(best_model, best_inliers_mask); + // run additional degeneracy check for F: + if (_degeneracy.dynamicCast()->verifyFundamental(best_model, best_score, best_inliers_mask, new_model, new_score)) { + // so-far-the-best F is degenerate + // Update best F using non-degenerate F or the one which is not from LO + if (new_score.isBetter(best_score_model_not_from_LO) && new_score.inlier_number > INL_THR*best_score.inlier_number) { + best_score = new_score; + new_model.copyTo(best_model); + } else if (best_score_model_not_from_LO.inlier_number > INL_THR*best_score.inlier_number) { + best_score = best_score_model_not_from_LO; + best_model_not_from_LO.copyTo(best_model); + } + } else { // so-far-the-best F is not degenerate + if (new_score.isBetter(best_score)) { + // if new model is better then update + best_score = new_score; + new_model.copyTo(best_model); + } + } + } + if (params->getFinalPolisher() != PolishingMethod::NONE_POLISHER) { Mat polished_model; Score polisher_score; - if (model_polisher->polishSoFarTheBestModel(best_model, best_score, - polished_model, polisher_score)) - if (polisher_score.isBetter(best_score)) { - best_score = polisher_score; - polished_model.copyTo(best_model); - } + if (polisher->polishSoFarTheBestModel(best_model, best_score, // polish final model + polished_model, polisher_score) && polisher_score.isBetter(best_score)) { + best_score = polisher_score; + polished_model.copyTo(best_model); + } } - // ================= here is ending ransac main implementation =========================== - std::vector inliers_mask; + + ///////////////// get inliers of the best model and points' residuals /////////////// + std::vector inliers_mask; std::vector residuals; if (params->isMaskRequired()) { inliers_mask = std::vector(points_size); - // get final inliers from the best model - _quality->getInliers(best_model, inliers_mask); + residuals = _error->getErrors(best_model); + _quality->getInliers(residuals, inliers_mask, threshold); } - // Store results - ransac_output = RansacOutput::create(best_model, inliers_mask, - static_cast(std::chrono::duration_cast - (std::chrono::steady_clock::now() - begin_time).count()), best_score.score, - best_score.inlier_number, final_iters, -1, -1); + + ModelConfidence model_conf = ModelConfidence::UNKNOWN; + if (IS_NON_RAND_TEST) { + std::vector temp_inliers(points_size); + const int non_random_inls_best_model = getIndependentInliers(best_model, best_sample, temp_inliers, + _quality->getInliers(best_model, temp_inliers)); + // quick test on lambda from all inliers (= upper bound of independent inliers) + // if model with independent inliers is not random for Poisson with all inliers then it is not random using independent inliers too + if (pow(Utils::getPoissonCDF(lambda_non_random_all_inliers, non_random_inls_best_model), num_total_tested_models) < 0.9999) { + std::vector inliers_list(models_for_random_test.size()); + for (int m = 0; m < (int)models_for_random_test.size(); m++) + inliers_list[m] = getIndependentInliers(models_for_random_test[m], samples_for_random_test[m], + temp_inliers, _quality->getInliers(models_for_random_test[m], temp_inliers)); + int min_non_rand_inliers; + const double lambda = getLambda(inliers_list, 1.644, points_size, sample_size, true, min_non_rand_inliers); + const double cdf_lambda = Utils::getPoissonCDF(lambda, non_random_inls_best_model), cdf_N = pow(cdf_lambda, num_total_tested_models); + model_conf = cdf_N < 0.9999 ? ModelConfidence ::RANDOM : ModelConfidence ::NON_RANDOM; + } else model_conf = ModelConfidence ::NON_RANDOM; + } + ransac_output = RansacOutput::create(best_model, inliers_mask, best_score.inlier_number, final_iters, model_conf, residuals); return true; } }; @@ -469,7 +1121,7 @@ void setParameters (int flag, Ptr ¶ms, EstimationMethod estimator, do params->setLocalOptimization(LocalOptimMethod ::LOCAL_OPTIM_INNER_LO); break; case USAC_FM_8PTS: - params = Model::create(thr, EstimationMethod::Fundamental8,SamplingMethod::SAMPLING_UNIFORM, + params = Model::create(thr, EstimationMethod::FUNDAMENTAL8,SamplingMethod::SAMPLING_UNIFORM, conf, max_iters,ScoreMethod::SCORE_METHOD_MSAC); params->setLocalOptimization(LocalOptimMethod ::LOCAL_OPTIM_INNER_LO); break; @@ -477,9 +1129,10 @@ void setParameters (int flag, Ptr ¶ms, EstimationMethod estimator, do } // do not do too many iterations for PnP if (estimator == EstimationMethod::P3P) { - if (params->getLOInnerMaxIters() > 15) - params->setLOIterations(15); + if (params->getLOInnerMaxIters() > 10) + params->setLOIterations(10); params->setLOIterativeIters(0); + params->setFinalLSQ(3); } params->maskRequired(mask_needed); @@ -488,9 +1141,9 @@ void setParameters (int flag, Ptr ¶ms, EstimationMethod estimator, do Mat findHomography (InputArray srcPoints, InputArray dstPoints, int method, double thr, OutputArray mask, const int max_iters, const double confidence) { Ptr params; - setParameters(method, params, EstimationMethod::Homography, thr, max_iters, confidence, mask.needed()); + setParameters(method, params, EstimationMethod::HOMOGRAPHY, thr, max_iters, confidence, mask.needed()); Ptr ransac_output; - if (run(params, srcPoints, dstPoints, params->getRandomGeneratorState(), + if (run(params, srcPoints, dstPoints, ransac_output, noArray(), noArray(), noArray(), noArray())) { saveMask(mask, ransac_output->getInliersMask()); return ransac_output->getModel() / ransac_output->getModel().at(2,2); @@ -505,9 +1158,9 @@ Mat findHomography (InputArray srcPoints, InputArray dstPoints, int method, doub Mat findFundamentalMat( InputArray points1, InputArray points2, int method, double thr, double confidence, int max_iters, OutputArray mask ) { Ptr params; - setParameters(method, params, EstimationMethod::Fundamental, thr, max_iters, confidence, mask.needed()); + setParameters(method, params, EstimationMethod::FUNDAMENTAL, thr, max_iters, confidence, mask.needed()); Ptr ransac_output; - if (run(params, points1, points2, params->getRandomGeneratorState(), + if (run(params, points1, points2, ransac_output, noArray(), noArray(), noArray(), noArray())) { saveMask(mask, ransac_output->getInliersMask()); return ransac_output->getModel(); @@ -522,9 +1175,9 @@ Mat findFundamentalMat( InputArray points1, InputArray points2, int method, doub Mat findEssentialMat (InputArray points1, InputArray points2, InputArray cameraMatrix1, int method, double prob, double thr, OutputArray mask, int maxIters) { Ptr params; - setParameters(method, params, EstimationMethod::Essential, thr, maxIters, prob, mask.needed()); + setParameters(method, params, EstimationMethod::ESSENTIAL, thr, maxIters, prob, mask.needed()); Ptr ransac_output; - if (run(params, points1, points2, params->getRandomGeneratorState(), + if (run(params, points1, points2, ransac_output, cameraMatrix1, cameraMatrix1, noArray(), noArray())) { saveMask(mask, ransac_output->getInliersMask()); return ransac_output->getModel(); @@ -544,7 +1197,7 @@ bool solvePnPRansac( InputArray objectPoints, InputArray imagePoints, setParameters(method, params, cameraMatrix.empty() ? EstimationMethod ::P6P : EstimationMethod ::P3P, thr, max_iters, conf, inliers.needed()); Ptr ransac_output; - if (run(params, imagePoints, objectPoints, params->getRandomGeneratorState(), + if (run(params, imagePoints, objectPoints, ransac_output, cameraMatrix, noArray(), distCoeffs, noArray())) { if (inliers.needed()) { const auto &inliers_mask = ransac_output->getInliersMask(); @@ -565,9 +1218,9 @@ bool solvePnPRansac( InputArray objectPoints, InputArray imagePoints, Mat estimateAffine2D(InputArray from, InputArray to, OutputArray mask, int method, double thr, int max_iters, double conf, int /*refineIters*/) { Ptr params; - setParameters(method, params, EstimationMethod ::Affine, thr, max_iters, conf, mask.needed()); + setParameters(method, params, EstimationMethod::AFFINE, thr, max_iters, conf, mask.needed()); Ptr ransac_output; - if (run(params, from, to, params->getRandomGeneratorState(), + if (run(params, from, to, ransac_output, noArray(), noArray(), noArray(), noArray())) { saveMask(mask, ransac_output->getInliersMask()); return ransac_output->getModel().rowRange(0,2); @@ -582,12 +1235,23 @@ Mat estimateAffine2D(InputArray from, InputArray to, OutputArray mask, int metho class ModelImpl : public Model { private: // main parameters: - double threshold, confidence; - int sample_size, max_iterations; - + double threshold; EstimationMethod estimator; SamplingMethod sampler; + double confidence; + int max_iterations; ScoreMethod score; + int sample_size; + + // Larsson parameters + bool is_larsson_optimization = true; + int larsson_leven_marq_iters_lo = 10, larsson_leven_marq_iters_fo = 15; + + // solver for a null-space extraction + MethodSolver null_solver = GEM_SOLVER; + + // prosac + int prosac_max_samples = 200000; // for neighborhood graph int k_nearest_neighbors = 8;//, flann_search_params = 5, num_kd_trees = 1; // for FLANN @@ -596,23 +1260,24 @@ private: NeighborSearchMethod neighborsType = NeighborSearchMethod::NEIGH_GRID; // Local Optimization parameters - LocalOptimMethod lo = LocalOptimMethod ::LOCAL_OPTIM_INNER_AND_ITER_LO; - int lo_sample_size=16, lo_inner_iterations=15, lo_iterative_iterations=8, - lo_thr_multiplier=15, lo_iter_sample_size = 30; + LocalOptimMethod lo = LocalOptimMethod ::LOCAL_OPTIM_INNER_LO; + int lo_sample_size=12, lo_inner_iterations=20, lo_iterative_iterations=8, + lo_thr_multiplier=10, lo_iter_sample_size = 30; // Graph cut parameters const double spatial_coherence_term = 0.975; // apply polisher for final RANSAC model - PolishingMethod polisher = PolishingMethod ::LSQPolisher; + PolishingMethod polisher = PolishingMethod ::COV_POLISHER; // preemptive verification test - VerificationMethod verifier = VerificationMethod ::SprtVerifier; - const int max_hypothesis_test_before_verification = 15; + VerificationMethod verifier = VerificationMethod ::ASPRT; // sprt parameters - // lower bound estimate is 1% of inliers - double sprt_eps = 0.01, sprt_delta = 0.008, avg_num_models, time_for_model_est; + // lower bound estimate is 2% of inliers + // model estimation to verification time = ratio of time needed to estimate model + // to verification of one point wrt the model + double sprt_eps = 0.02, sprt_delta = 0.008, avg_num_models, model_est_to_ver_time; // estimator error ErrorMetric est_error; @@ -620,67 +1285,71 @@ private: // progressive napsac double relax_coef = 0.1; // for building neighborhood graphs - const std::vector grid_cell_number = {16, 8, 4, 2}; + const std::vector grid_cell_number = {10, 5, 2}; //for final least squares polisher - int final_lsq_iters = 3; + int final_lsq_iters = 7; - bool need_mask = true, is_parallel = false; + bool need_mask = true, // do we need inlier mask in the end + is_parallel = false, // use parallel RANSAC + is_nonrand_test = false; // is test for the final model non-randomness + + // state of pseudo-random number generator int random_generator_state = 0; - const int max_iters_before_LO = 100; + + // number of iterations of plane-and-parallax in DEGENSAC^+ + int plane_and_parallax_max_iters = 300; // magsac parameters: int DoF = 2; double sigma_quantile = 3.04, upper_incomplete_of_sigma_quantile = 0.00419, - lower_incomplete_of_sigma_quantile = 0.8629, C = 0.5, maximum_thr = 7.5; + lower_incomplete_of_sigma_quantile = 0.8629, C = 0.5, maximum_thr = 7.5; + double k_mlesac = 2.25; // parameter for MLESAC model evaluation public: - ModelImpl (double threshold_, EstimationMethod estimator_, SamplingMethod sampler_, double confidence_=0.95, - int max_iterations_=5000, ScoreMethod score_ =ScoreMethod::SCORE_METHOD_MSAC) { - estimator = estimator_; - sampler = sampler_; - confidence = confidence_; - max_iterations = max_iterations_; - score = score_; - + ModelImpl (double threshold_, EstimationMethod estimator_, SamplingMethod sampler_, double confidence_, + int max_iterations_, ScoreMethod score_) : + threshold(threshold_), estimator(estimator_), sampler(sampler_), confidence(confidence_), max_iterations(max_iterations_), score(score_) { switch (estimator_) { - // time for model estimation is basically a ratio of time need to estimate a model to - // time needed to verify if a point is consistent with this model - case (EstimationMethod::Affine): - avg_num_models = 1; time_for_model_est = 50; + case (EstimationMethod::AFFINE): + avg_num_models = 1; model_est_to_ver_time = 50; sample_size = 3; est_error = ErrorMetric ::FORW_REPR_ERR; break; - case (EstimationMethod::Homography): - avg_num_models = 1; time_for_model_est = 150; + case (EstimationMethod::HOMOGRAPHY): + avg_num_models = 0.8; model_est_to_ver_time = 200; sample_size = 4; est_error = ErrorMetric ::FORW_REPR_ERR; break; - case (EstimationMethod::Fundamental): - avg_num_models = 2.38; time_for_model_est = 180; maximum_thr = 2.5; + case (EstimationMethod::FUNDAMENTAL): + DoF = 4; C = 0.25; sigma_quantile = 3.64, upper_incomplete_of_sigma_quantile = 0.003657; lower_incomplete_of_sigma_quantile = 1.3012; + maximum_thr = 2.5; + avg_num_models = 1.5; model_est_to_ver_time = 200; sample_size = 7; est_error = ErrorMetric ::SAMPSON_ERR; break; - case (EstimationMethod::Fundamental8): - avg_num_models = 1; time_for_model_est = 100; maximum_thr = 2.5; + case (EstimationMethod::FUNDAMENTAL8): + avg_num_models = 1; model_est_to_ver_time = 100; maximum_thr = 2.5; sample_size = 8; est_error = ErrorMetric ::SAMPSON_ERR; break; - case (EstimationMethod::Essential): - avg_num_models = 3.93; time_for_model_est = 1000; maximum_thr = 2.5; - sample_size = 5; est_error = ErrorMetric ::SGD_ERR; break; + case (EstimationMethod::ESSENTIAL): + DoF = 4; C = 0.25; sigma_quantile = 3.64, upper_incomplete_of_sigma_quantile = 0.003657; lower_incomplete_of_sigma_quantile = 1.3012; + avg_num_models = 3.93; model_est_to_ver_time = 1000; maximum_thr = 2; + sample_size = 5; est_error = ErrorMetric ::SAMPSON_ERR; break; case (EstimationMethod::P3P): - avg_num_models = 1.38; time_for_model_est = 800; + avg_num_models = 1.38; model_est_to_ver_time = 800; sample_size = 3; est_error = ErrorMetric ::RERPOJ; break; case (EstimationMethod::P6P): - avg_num_models = 1; time_for_model_est = 300; + avg_num_models = 1; model_est_to_ver_time = 300; sample_size = 6; est_error = ErrorMetric ::RERPOJ; break; default: CV_Error(cv::Error::StsNotImplemented, "Estimator has not implemented yet!"); } + if (score_ == ScoreMethod::SCORE_METHOD_MAGSAC) + polisher = PolishingMethod::MAGSAC; + + // for PnP problem we can use only KNN graph if (estimator_ == EstimationMethod::P3P || estimator_ == EstimationMethod::P6P) { + polisher = LSQ_POLISHER; neighborsType = NeighborSearchMethod::NEIGH_FLANN_KNN; k_nearest_neighbors = 2; } - if (estimator == EstimationMethod::Fundamental || estimator == EstimationMethod::Essential) { - lo_sample_size = 21; - lo_thr_multiplier = 10; - } - if (estimator == EstimationMethod::Homography) - maximum_thr = 8.; - threshold = threshold_; } + + // setters + void setNonRandomnessTest (bool set) override { is_nonrand_test = set; } void setVerifier (VerificationMethod verifier_) override { verifier = verifier_; } void setPolisher (PolishingMethod polisher_) override { polisher = polisher_; } void setParallel (bool is_parallel_) override { is_parallel = is_parallel_; } @@ -690,11 +1359,17 @@ public: void setNeighborsType (NeighborSearchMethod neighbors) override { neighborsType = neighbors; } void setCellSize (int cell_size_) override { cell_size = cell_size_; } void setLOIterations (int iters) override { lo_inner_iterations = iters; } - void setLOIterativeIters (int iters) override {lo_iterative_iterations = iters; } void setLOSampleSize (int lo_sample_size_) override { lo_sample_size = lo_sample_size_; } - void setThresholdMultiplierLO (double thr_mult) override { lo_thr_multiplier = (int) round(thr_mult); } void maskRequired (bool need_mask_) override { need_mask = need_mask_; } void setRandomGeneratorState (int state) override { random_generator_state = state; } + void setLOIterativeIters (int iters) override { lo_iterative_iterations = iters; } + void setFinalLSQ (int iters) override { final_lsq_iters = iters; } + + // getters + int getProsacMaxSamples() const override { return prosac_max_samples; } + int getLevMarqIters () const override { return larsson_leven_marq_iters_fo; } + int getLevMarqItersLO () const override { return larsson_leven_marq_iters_lo; } + bool isNonRandomnessTest () const override { return is_nonrand_test; } bool isMaskRequired () const override { return need_mask; } NeighborSearchMethod getNeighborsSearch () const override { return neighborsType; } int getKNN () const override { return k_nearest_neighbors; } @@ -711,17 +1386,17 @@ public: return lower_incomplete_of_sigma_quantile; } double getC () const override { return C; } + double getKmlesac () const override { return k_mlesac; } double getMaximumThreshold () const override { return maximum_thr; } double getGraphCutSpatialCoherenceTerm () const override { return spatial_coherence_term; } int getLOSampleSize () const override { return lo_sample_size; } - int getMaxNumHypothesisToTestBeforeRejection() const override { - return max_hypothesis_test_before_verification; - } + MethodSolver getRansacSolver () const override { return null_solver; } PolishingMethod getFinalPolisher () const override { return polisher; } int getLOThresholdMultiplier() const override { return lo_thr_multiplier; } int getLOIterativeSampleSize() const override { return lo_iter_sample_size; } int getLOIterativeMaxIters() const override { return lo_iterative_iterations; } int getLOInnerMaxIters() const override { return lo_inner_iterations; } + int getPlaneAndParallaxIters () const override { return plane_and_parallax_max_iters; } LocalOptimMethod getLO () const override { return lo; } ScoreMethod getScore () const override { return score; } int getMaxIters () const override { return max_iterations; } @@ -730,22 +1405,22 @@ public: VerificationMethod getVerifier () const override { return verifier; } SamplingMethod getSampler () const override { return sampler; } int getRandomGeneratorState () const override { return random_generator_state; } - int getMaxItersBeforeLO () const override { return max_iters_before_LO; } double getSPRTdelta () const override { return sprt_delta; } double getSPRTepsilon () const override { return sprt_eps; } double getSPRTavgNumModels () const override { return avg_num_models; } int getCellSize () const override { return cell_size; } int getGraphRadius() const override { return radius; } - double getTimeForModelEstimation () const override { return time_for_model_est; } + double getTimeForModelEstimation () const override { return model_est_to_ver_time; } double getRelaxCoef () const override { return relax_coef; } const std::vector &getGridCellNumber () const override { return grid_cell_number; } + bool isLarssonOptimization () const override { return is_larsson_optimization; } bool isParallel () const override { return is_parallel; } bool isFundamental () const override { - return estimator == EstimationMethod ::Fundamental || - estimator == EstimationMethod ::Fundamental8; + return estimator == EstimationMethod::FUNDAMENTAL || + estimator == EstimationMethod::FUNDAMENTAL8; } - bool isHomography () const override { return estimator == EstimationMethod ::Homography; } - bool isEssential () const override { return estimator == EstimationMethod ::Essential; } + bool isHomography () const override { return estimator == EstimationMethod::HOMOGRAPHY; } + bool isEssential () const override { return estimator == EstimationMethod::ESSENTIAL; } bool isPnP() const override { return estimator == EstimationMethod ::P3P || estimator == EstimationMethod ::P6P; } @@ -757,277 +1432,36 @@ Ptr Model::create(double threshold_, EstimationMethod estimator_, Samplin max_iterations_, score_); } -bool run (const Ptr ¶ms, InputArray points1, InputArray points2, int state, +bool run (const Ptr ¶ms, InputArray points1, InputArray points2, Ptr &ransac_output, InputArray K1_, InputArray K2_, InputArray dist_coeff1, InputArray dist_coeff2) { - Ptr error; - Ptr estimator; - Ptr graph; - Ptr degeneracy; - Ptr quality; - Ptr verifier; - Ptr sampler; - Ptr lo_sampler; - Ptr termination; - Ptr lo; - Ptr polisher; - Ptr min_solver; - Ptr non_min_solver; - - Mat points, calib_points, undist_points1, undist_points2; - Matx33d K1, K2; - int points_size; - double threshold = params->getThreshold(), max_thr = params->getMaximumThreshold(); - const int min_sample_size = params->getSampleSize(); - if (params->isPnP()) { - if (! K1_.empty()) { - K1 = K1_.getMat(); - if (! dist_coeff1.empty()) { - // undistortPoints also calibrate points using K - if (points1.isContinuous()) - undistortPoints(points1, undist_points1, K1_, dist_coeff1); - else undistortPoints(points1.getMat().clone(), undist_points1, K1_, dist_coeff1); - points_size = mergePoints(undist_points1, points2, points, true); - Utils::normalizeAndDecalibPointsPnP (K1, points, calib_points); - } else { - points_size = mergePoints(points1, points2, points, true); - Utils::calibrateAndNormalizePointsPnP(K1, points, calib_points); - } - } else - points_size = mergePoints(points1, points2, points, true); - } else { - if (params->isEssential()) { - CV_CheckEQ((int)(!K1_.empty() && !K2_.empty()), 1, "Intrinsic matrix must not be empty!"); - K1 = K1_.getMat(); - K2 = K2_.getMat(); - if (! dist_coeff1.empty() || ! dist_coeff2.empty()) { - // undistortPoints also calibrate points using K - if (points1.isContinuous()) - undistortPoints(points1, undist_points1, K1_, dist_coeff1); - else undistortPoints(points1.getMat().clone(), undist_points1, K1_, dist_coeff1); - if (points2.isContinuous()) - undistortPoints(points2, undist_points2, K2_, dist_coeff2); - else undistortPoints(points2.getMat().clone(), undist_points2, K2_, dist_coeff2); - points_size = mergePoints(undist_points1, undist_points2, calib_points, false); - } else { - points_size = mergePoints(points1, points2, points, false); - Utils::calibratePoints(K1, K2, points, calib_points); - } - threshold = Utils::getCalibratedThreshold(threshold, K1, K2); - max_thr = Utils::getCalibratedThreshold(max_thr, K1, K2); - } else - points_size = mergePoints(points1, points2, points, false); - } - - // Since error function output squared error distance, so make - // threshold squared as well - threshold *= threshold; - - if (params->getSampler() == SamplingMethod::SAMPLING_NAPSAC || params->getLO() == LocalOptimMethod::LOCAL_OPTIM_GC) { - if (params->getNeighborsSearch() == NeighborSearchMethod::NEIGH_GRID) { - graph = GridNeighborhoodGraph::create(points, points_size, - params->getCellSize(), params->getCellSize(), - params->getCellSize(), params->getCellSize(), 10); - } else if (params->getNeighborsSearch() == NeighborSearchMethod::NEIGH_FLANN_KNN) { - graph = FlannNeighborhoodGraph::create(points, points_size,params->getKNN(), false, 5, 1); - } else if (params->getNeighborsSearch() == NeighborSearchMethod::NEIGH_FLANN_RADIUS) { - graph = RadiusSearchNeighborhoodGraph::create(points, points_size, - params->getGraphRadius(), 5, 1); - } else CV_Error(cv::Error::StsNotImplemented, "Graph type is not implemented!"); - } - - std::vector> layers; - if (params->getSampler() == SamplingMethod::SAMPLING_PROGRESSIVE_NAPSAC) { - CV_CheckEQ((int)params->isPnP(), 0, "ProgressiveNAPSAC for PnP is not implemented!"); - const auto &cell_number_per_layer = params->getGridCellNumber(); - layers.reserve(cell_number_per_layer.size()); - const auto * const pts = (float *) points.data; - float img1_width = 0, img1_height = 0, img2_width = 0, img2_height = 0; - for (int i = 0; i < 4 * points_size; i += 4) { - if (pts[i ] > img1_width ) img1_width = pts[i ]; - if (pts[i + 1] > img1_height) img1_height = pts[i + 1]; - if (pts[i + 2] > img2_width ) img2_width = pts[i + 2]; - if (pts[i + 3] > img2_height) img2_height = pts[i + 3]; - } - // Create grid graphs (overlapping layes of given cell numbers) - for (int layer_idx = 0; layer_idx < (int)cell_number_per_layer.size(); layer_idx++) { - const int cell_number = cell_number_per_layer[layer_idx]; - if (layer_idx > 0) - if (cell_number_per_layer[layer_idx-1] <= cell_number) - CV_Error(cv::Error::StsError, "Progressive NAPSAC sampler: " - "Cell number in layers must be in decreasing order!"); - layers.emplace_back(GridNeighborhoodGraph::create(points, points_size, - (int)(img1_width / (float)cell_number), (int)(img1_height / (float)cell_number), - (int)(img2_width / (float)cell_number), (int)(img2_height / (float)cell_number), 10)); - } - } - - // update points by calibrated for Essential matrix after graph is calculated - if (params->isEssential()) { - points = calib_points; - // if maximum calibrated threshold significanlty differs threshold then set upper bound - if (max_thr > 10*threshold) - max_thr = sqrt(10*threshold); // max thr will be squared after - } - if (max_thr < threshold) - max_thr = threshold; - - switch (params->getError()) { - case ErrorMetric::SYMM_REPR_ERR: - error = ReprojectionErrorSymmetric::create(points); break; - case ErrorMetric::FORW_REPR_ERR: - if (params->getEstimator() == EstimationMethod::Affine) - error = ReprojectionErrorAffine::create(points); - else error = ReprojectionErrorForward::create(points); - break; - case ErrorMetric::SAMPSON_ERR: - error = SampsonError::create(points); break; - case ErrorMetric::SGD_ERR: - error = SymmetricGeometricDistance::create(points); break; - case ErrorMetric::RERPOJ: - error = ReprojectionErrorPmatrix::create(points); break; - default: CV_Error(cv::Error::StsNotImplemented , "Error metric is not implemented!"); - } - - switch (params->getScore()) { - case ScoreMethod::SCORE_METHOD_RANSAC : - quality = RansacQuality::create(points_size, threshold, error); break; - case ScoreMethod::SCORE_METHOD_MSAC : - quality = MsacQuality::create(points_size, threshold, error); break; - case ScoreMethod::SCORE_METHOD_MAGSAC : - quality = MagsacQuality::create(max_thr, points_size, error, - threshold, params->getDegreesOfFreedom(), params->getSigmaQuantile(), - params->getUpperIncompleteOfSigmaQuantile(), - params->getLowerIncompleteOfSigmaQuantile(), params->getC()); break; - case ScoreMethod::SCORE_METHOD_LMEDS : - quality = LMedsQuality::create(points_size, threshold, error); break; - default: CV_Error(cv::Error::StsNotImplemented, "Score is not imeplemeted!"); - } - - if (params->isHomography()) { - degeneracy = HomographyDegeneracy::create(points); - min_solver = HomographyMinimalSolver4ptsGEM::create(points); - non_min_solver = HomographyNonMinimalSolver::create(points); - estimator = HomographyEstimator::create(min_solver, non_min_solver, degeneracy); - } else if (params->isFundamental()) { - degeneracy = FundamentalDegeneracy::create(state++, quality, points, min_sample_size, 5. /*sqr homogr thr*/); - if(min_sample_size == 7) min_solver = FundamentalMinimalSolver7pts::create(points); - else min_solver = FundamentalMinimalSolver8pts::create(points); - non_min_solver = FundamentalNonMinimalSolver::create(points); - estimator = FundamentalEstimator::create(min_solver, non_min_solver, degeneracy); - } else if (params->isEssential()) { - degeneracy = EssentialDegeneracy::create(points, min_sample_size); - min_solver = EssentialMinimalSolverStewenius5pts::create(points); - non_min_solver = EssentialNonMinimalSolver::create(points); - estimator = EssentialEstimator::create(min_solver, non_min_solver, degeneracy); - } else if (params->isPnP()) { - degeneracy = makePtr(); - if (min_sample_size == 3) { - non_min_solver = DLSPnP::create(points, calib_points, K1); - min_solver = P3PSolver::create(points, calib_points, K1); - } else { - min_solver = PnPMinimalSolver6Pts::create(points); - non_min_solver = PnPNonMinimalSolver::create(points); - } - estimator = PnPEstimator::create(min_solver, non_min_solver); - } else if (params->getEstimator() == EstimationMethod::Affine) { - degeneracy = makePtr(); - min_solver = AffineMinimalSolver::create(points); - non_min_solver = AffineNonMinimalSolver::create(points); - estimator = AffineEstimator::create(min_solver, non_min_solver); - } else CV_Error(cv::Error::StsNotImplemented, "Estimator not implemented!"); - - switch (params->getSampler()) { - case SamplingMethod::SAMPLING_UNIFORM: - sampler = UniformSampler::create(state++, min_sample_size, points_size); break; - case SamplingMethod::SAMPLING_PROSAC: - sampler = ProsacSampler::create(state++, points_size, min_sample_size, 200000); break; - case SamplingMethod::SAMPLING_PROGRESSIVE_NAPSAC: - sampler = ProgressiveNapsac::create(state++, points_size, min_sample_size, layers, 20); break; - case SamplingMethod::SAMPLING_NAPSAC: - sampler = NapsacSampler::create(state++, points_size, min_sample_size, graph); break; - default: CV_Error(cv::Error::StsNotImplemented, "Sampler is not implemented!"); - } - - switch (params->getVerifier()) { - case VerificationMethod::NullVerifier: verifier = ModelVerifier::create(); break; - case VerificationMethod::SprtVerifier: - verifier = SPRT::create(state++, error, points_size, params->getScore() == ScoreMethod ::SCORE_METHOD_MAGSAC ? max_thr : threshold, - params->getSPRTepsilon(), params->getSPRTdelta(), params->getTimeForModelEstimation(), - params->getSPRTavgNumModels(), params->getScore()); break; - default: CV_Error(cv::Error::StsNotImplemented, "Verifier is not imeplemented!"); - } - - if (params->getSampler() == SamplingMethod::SAMPLING_PROSAC) { - termination = ProsacTerminationCriteria::create(sampler.dynamicCast(), error, - points_size, min_sample_size, params->getConfidence(), - params->getMaxIters(), 100, 0.05, 0.05, threshold); - } else if (params->getSampler() == SamplingMethod::SAMPLING_PROGRESSIVE_NAPSAC) { - if (params->getVerifier() == VerificationMethod::SprtVerifier) - termination = SPRTPNapsacTermination::create(((SPRT *)verifier.get())->getSPRTvector(), - params->getConfidence(), points_size, min_sample_size, - params->getMaxIters(), params->getRelaxCoef()); - else - termination = StandardTerminationCriteria::create (params->getConfidence(), - points_size, min_sample_size, params->getMaxIters()); - } else if (params->getVerifier() == VerificationMethod::SprtVerifier) { - termination = SPRTTermination::create(((SPRT *) verifier.get())->getSPRTvector(), - params->getConfidence(), points_size, min_sample_size, params->getMaxIters()); - } else - termination = StandardTerminationCriteria::create - (params->getConfidence(), points_size, min_sample_size, params->getMaxIters()); - - if (params->getLO() != LocalOptimMethod::LOCAL_OPTIM_NULL) { - lo_sampler = UniformRandomGenerator::create(state++, points_size, params->getLOSampleSize()); - switch (params->getLO()) { - case LocalOptimMethod::LOCAL_OPTIM_INNER_LO: - lo = InnerIterativeLocalOptimization::create(estimator, quality, lo_sampler, - points_size, threshold, false, params->getLOIterativeSampleSize(), - params->getLOInnerMaxIters(), params->getLOIterativeMaxIters(), - params->getLOThresholdMultiplier()); break; - case LocalOptimMethod::LOCAL_OPTIM_INNER_AND_ITER_LO: - lo = InnerIterativeLocalOptimization::create(estimator, quality, lo_sampler, - points_size, threshold, true, params->getLOIterativeSampleSize(), - params->getLOInnerMaxIters(), params->getLOIterativeMaxIters(), - params->getLOThresholdMultiplier()); break; - case LocalOptimMethod::LOCAL_OPTIM_GC: - lo = GraphCut::create(estimator, error, quality, graph, lo_sampler, threshold, - params->getGraphCutSpatialCoherenceTerm(), params->getLOInnerMaxIters()); break; - case LocalOptimMethod::LOCAL_OPTIM_SIGMA: - lo = SigmaConsensus::create(estimator, error, quality, verifier, - params->getLOSampleSize(), params->getLOInnerMaxIters(), - params->getDegreesOfFreedom(), params->getSigmaQuantile(), - params->getUpperIncompleteOfSigmaQuantile(), params->getC(), max_thr); break; - default: CV_Error(cv::Error::StsNotImplemented , "Local Optimization is not implemented!"); - } - } - - if (params->getFinalPolisher() == PolishingMethod::LSQPolisher) - polisher = LeastSquaresPolishing::create(estimator, quality, params->getFinalLSQIterations()); - - Ransac ransac (params, points_size, estimator, quality, sampler, - termination, verifier, degeneracy, lo, polisher, params->isParallel(), state); + Ransac ransac (params, points1, points2, K1_, K2_, dist_coeff1, dist_coeff2); if (ransac.run(ransac_output)) { if (params->isPnP()) { // convert R to rodrigues and back and recalculate inliers which due to numerical // issues can differ - Mat out, R, newR, newP, t, rvec; + Mat out, newP; + Matx33d R, newR, K1; + Vec3d t, rvec; if (K1_.empty()) { usac::Utils::decomposeProjection (ransac_output->getModel(), K1, R, t); Rodrigues(R, rvec); hconcat(rvec, t, out); hconcat(out, K1, out); } else { - const Mat Rt = K1.inv() * ransac_output->getModel(); + K1 = ransac.K1; + const Mat Rt = Mat(Matx33d(K1).inv() * Matx34d(ransac_output->getModel())); t = Rt.col(3); Rodrigues(Rt.colRange(0,3), rvec); hconcat(rvec, t, out); } + // Matx33d _K1(K1); Rodrigues(rvec, newR); - hconcat(K1 * newR, K1 * t, newP); - std::vector inliers_mask(points_size); - quality->getInliers(newP, inliers_mask); - ransac_output = RansacOutput::create(out, inliers_mask, 0,0,0,0,0,0); + hconcat(K1 * Matx33d(newR), K1 * Vec3d(t), newP); + std::vector inliers_mask(ransac.points_size); + ransac._quality->getInliers(newP, inliers_mask); + ransac_output = RansacOutput::create(out, inliers_mask, ransac_output->getNumberOfInliers(), + ransac_output->getNumberOfIters(), ransac_output->getConfidence(), ransac_output->getResiduals()); } return true; } diff --git a/modules/calib3d/src/usac/sampler.cpp b/modules/calib3d/src/usac/sampler.cpp index c44372853e..2095ee8b4d 100644 --- a/modules/calib3d/src/usac/sampler.cpp +++ b/modules/calib3d/src/usac/sampler.cpp @@ -40,9 +40,6 @@ public: points_random_pool[--random_pool_size]); } } - Ptr clone (int state) const override { - return makePtr(state, sample_size, points_size); - } private: void setPointsSize (int points_size_) { CV_Assert (sample_size <= points_size_); @@ -143,10 +140,6 @@ public: points_size = points_size_; initialize (); } - Ptr clone (int state) const override { - return makePtr(state, points_size, sample_size, - max_prosac_samples_count); - } private: void initialize () { largest_sample_size = points_size; // termination length, n* @@ -266,15 +259,19 @@ public: // Choice of the hypothesis generation set // if (t = T'_n) & (n < n*) then n = n + 1 (eqn. 4) - if (kth_sample_number == growth_function[subset_size-1] && subset_size < termination_length) + if (kth_sample_number >= growth_function[subset_size-1] && subset_size < termination_length) subset_size++; // Semi-random sample M_t of size m // if T'n < t then if (growth_function[subset_size-1] < kth_sample_number) { - // The sample contains m-1 points selected from U_(n-1) at random and u_n - random_gen->generateUniqueRandomSet(sample, sample_size-1, subset_size-1); - sample[sample_size-1] = subset_size-1; + if (subset_size >= termination_length) { + random_gen->generateUniqueRandomSet(sample, sample_size, subset_size); + } else { + // The sample contains m-1 points selected from U_(n-1) at random and u_n + random_gen->generateUniqueRandomSet(sample, sample_size-1, subset_size-1); + sample[sample_size-1] = subset_size-1; + } } else { // Select m points from U_n at random. random_gen->generateUniqueRandomSet(sample, sample_size, subset_size); @@ -306,10 +303,6 @@ public: CV_Error(cv::Error::StsError, "Changing points size in PROSAC requires to change also " "termination criteria! Use PROSAC simpler version"); } - Ptr clone (int state) const override { - return makePtr(state, points_size, sample_size, - growth_max_samples); - } }; Ptr ProsacSampler::create(int state, int points_size_, int sample_size_, @@ -465,10 +458,6 @@ public: CV_Error(cv::Error::StsError, "Changing points size requires changing neighborhood graph! " "You must reinitialize P-NAPSAC!"); } - Ptr clone (int state) const override { - return makePtr(state, points_size, sample_size, *layers, - sampler_length); - } }; Ptr ProgressiveNapsac::create(int state, int points_size_, int sample_size_, const std::vector> &layers, int sampler_length_) { @@ -537,9 +526,6 @@ public: CV_Error(cv::Error::StsError, "Changing points size requires changing neighborhood graph!" " You must reinitialize NAPSAC!"); } - Ptr clone (int state) const override { - return makePtr(state, points_size, sample_size, neighborhood_graph); - } }; Ptr NapsacSampler::create(int state, int points_size_, int sample_size_, const Ptr &neighborhood_graph_) { diff --git a/modules/calib3d/src/usac/termination.cpp b/modules/calib3d/src/usac/termination.cpp index 6c341a9366..803b060e41 100644 --- a/modules/calib3d/src/usac/termination.cpp +++ b/modules/calib3d/src/usac/termination.cpp @@ -28,7 +28,8 @@ public: * (1 - w^n) is probability that at least one point of N is outlier. * 1 - p = (1-w^n)^k is probability that in K steps of getting at least one outlier is 1% (5%). */ - int update (const Mat &/*model*/, int inlier_number) override { + int update (const Mat &model, int inlier_number) const override { + CV_UNUSED(model); const double predicted_iters = log_confidence / log(1 - std::pow (static_cast(inlier_number) / points_size, sample_size)); @@ -40,9 +41,11 @@ public: return MAX_ITERATIONS; } - Ptr clone () const override { - return makePtr(1-exp(log_confidence), points_size, - sample_size, MAX_ITERATIONS); + static int getMaxIterations (int inlier_number, int sample_size, int points_size, double conf) { + const double pred_iters = log(1 - conf) / log(1 - pow(static_cast(inlier_number)/points_size, sample_size)); + if (std::isinf(pred_iters)) + return INT_MAX; + return (int) pred_iters + 1; } }; Ptr StandardTerminationCriteria::create(double confidence, @@ -54,13 +57,13 @@ Ptr StandardTerminationCriteria::create(double conf /////////////////////////////////////// SPRT TERMINATION ////////////////////////////////////////// class SPRTTerminationImpl : public SPRTTermination { private: - const std::vector &sprt_histories; + const Ptr sprt; const double log_eta_0; const int points_size, sample_size, MAX_ITERATIONS; public: - SPRTTerminationImpl (const std::vector &sprt_histories_, double confidence, + SPRTTerminationImpl (const Ptr &sprt_, double confidence, int points_size_, int sample_size_, int max_iterations_) - : sprt_histories (sprt_histories_), log_eta_0(log(1-confidence)), + : sprt (sprt_), log_eta_0(log(1-confidence)), points_size (points_size_), sample_size (sample_size_),MAX_ITERATIONS(max_iterations_){} /* @@ -80,9 +83,11 @@ public: * this equation does not have to be evaluated before nR < n0 * nR = (1 - P_g)^k */ - int update (const Mat &/*model*/, int inlier_size) override { - if (sprt_histories.empty()) - return std::min(MAX_ITERATIONS, getStandardUpperBound(inlier_size)); + int update (const Mat &model, int inlier_size) const override { + CV_UNUSED(model); + const auto &sprt_histories = sprt->getSPRTvector(); + if (sprt_histories.size() <= 1) + return getStandardUpperBound(inlier_size); const double epsilon = static_cast(inlier_size) / points_size; // inlier probability const double P_g = pow (epsilon, sample_size); // probability of good sample @@ -90,23 +95,21 @@ public: double log_eta_lmin1 = 0; int total_number_of_tested_samples = 0; - const int sprts_size_min1 = static_cast(sprt_histories.size())-1; - if (sprts_size_min1 < 0) return getStandardUpperBound(inlier_size); // compute log n(l-1), l is number of tests - for (int test = 0; test < sprts_size_min1; test++) { - log_eta_lmin1 += log (1 - P_g * (1 - pow (sprt_histories[test].A, - -computeExponentH(sprt_histories[test].epsilon, epsilon,sprt_histories[test].delta)))) - * sprt_histories[test].tested_samples; - total_number_of_tested_samples += sprt_histories[test].tested_samples; + for (const auto &test : sprt_histories) { + if (test.tested_samples == 0) continue; + log_eta_lmin1 += log (1 - P_g * (1 - pow (test.A, + -computeExponentH(test.epsilon, epsilon,test.delta)))) * test.tested_samples; + total_number_of_tested_samples += test.tested_samples; } // Implementation note: since η > ηR the equation (9) does not have to be evaluated // before ηR < η0 is satisfied. if (std::pow(1 - P_g, total_number_of_tested_samples) < log_eta_0) - return std::min(MAX_ITERATIONS, getStandardUpperBound(inlier_size)); + return getStandardUpperBound(inlier_size); // use decision threshold A for last test (l-th) - const double predicted_iters_sprt = (log_eta_0 - log_eta_lmin1) / - log (1 - P_g * (1 - 1 / sprt_histories[sprts_size_min1].A)); // last A + const double predicted_iters_sprt = total_number_of_tested_samples + (log_eta_0 - log_eta_lmin1) / + log (1 - P_g * (1 - 1 / sprt_histories.back().A)); // last A if (std::isnan(predicted_iters_sprt) || std::isinf(predicted_iters_sprt)) return getStandardUpperBound(inlier_size); @@ -117,11 +120,6 @@ public: getStandardUpperBound(inlier_size)); return getStandardUpperBound(inlier_size); } - - Ptr clone () const override { - return makePtr(sprt_histories, 1-exp(log_eta_0), points_size, - sample_size, MAX_ITERATIONS); - } private: inline int getStandardUpperBound(int inlier_size) const { const double predicted_iters = log_eta_0 / log(1 - std::pow @@ -157,9 +155,9 @@ private: return h; } }; -Ptr SPRTTermination::create(const std::vector &sprt_histories_, +Ptr SPRTTermination::create(const Ptr &sprt_, double confidence, int points_size_, int sample_size_, int max_iterations_) { - return makePtr(sprt_histories_, confidence, points_size_, sample_size_, + return makePtr(sprt_, confidence, points_size_, sample_size_, max_iterations_); } @@ -167,21 +165,20 @@ Ptr SPRTTermination::create(const std::vector &sp class SPRTPNapsacTerminationImpl : public SPRTPNapsacTermination { private: SPRTTerminationImpl sprt_termination; - const std::vector &sprt_histories; const double relax_coef, log_confidence; const int points_size, sample_size, MAX_ITERS; public: - SPRTPNapsacTerminationImpl (const std::vector &sprt_histories_, + SPRTPNapsacTerminationImpl (const Ptr &sprt, double confidence, int points_size_, int sample_size_, int max_iterations_, double relax_coef_) - : sprt_termination (sprt_histories_, confidence, points_size_, sample_size_, - max_iterations_), sprt_histories (sprt_histories_), + : sprt_termination (sprt, confidence, points_size_, sample_size_, + max_iterations_), relax_coef (relax_coef_), log_confidence(log(1-confidence)), points_size (points_size_), sample_size (sample_size_), MAX_ITERS (max_iterations_) {} - int update (const Mat &model, int inlier_number) override { + int update (const Mat &model, int inlier_number) const override { int predicted_iterations = sprt_termination.update(model, inlier_number); const double inlier_prob = static_cast(inlier_number) / points_size + relax_coef; @@ -192,52 +189,41 @@ public: if (! std::isinf(predicted_iters) && predicted_iters < predicted_iterations) return static_cast(predicted_iters); - return predicted_iterations; - } - Ptr clone () const override { - return makePtr(sprt_histories, 1-exp(log_confidence), - points_size, sample_size, MAX_ITERS, relax_coef); + return std::min(MAX_ITERS, predicted_iterations); } }; -Ptr SPRTPNapsacTermination::create(const std::vector& - sprt_histories_, double confidence, int points_size_, int sample_size_, +Ptr SPRTPNapsacTermination::create(const Ptr & + sprt, double confidence, int points_size_, int sample_size_, int max_iterations_, double relax_coef_) { - return makePtr(sprt_histories_, confidence, points_size_, + return makePtr(sprt, confidence, points_size_, sample_size_, max_iterations_, relax_coef_); } -////////////////////////////////////// PROSAC TERMINATION ///////////////////////////////////////// +////////////////////////////////////// PROSAC TERMINATION ///////////////////////////////////////// class ProsacTerminationCriteriaImpl : public ProsacTerminationCriteria { private: - const double log_confidence, beta, non_randomness_phi, inlier_threshold; + const double log_conf, beta, non_randomness_phi, inlier_threshold; const int MAX_ITERATIONS, points_size, min_termination_length, sample_size; const Ptr sampler; - std::vector non_random_inliers; - const Ptr error; public: - ProsacTerminationCriteriaImpl (const Ptr &error_, int points_size_,int sample_size_, - double confidence, int max_iterations, int min_termination_length_, double beta_, - double non_randomness_phi_, double inlier_threshold_) : log_confidence - (log(1-confidence)), beta(beta_), non_randomness_phi(non_randomness_phi_), - inlier_threshold(inlier_threshold_), MAX_ITERATIONS(max_iterations), - points_size (points_size_), min_termination_length (min_termination_length_), - sample_size(sample_size_), error (error_) { init(); } - ProsacTerminationCriteriaImpl (const Ptr &sampler_,const Ptr &error_, int points_size_, int sample_size_, double confidence, int max_iterations, int min_termination_length_, double beta_, double non_randomness_phi_, - double inlier_threshold_) : log_confidence(log(1-confidence)), beta(beta_), + double inlier_threshold_, const std::vector &non_rand_inliers) : log_conf(log(1-confidence)), beta(beta_), non_randomness_phi(non_randomness_phi_), inlier_threshold(inlier_threshold_), MAX_ITERATIONS(max_iterations), points_size (points_size_), min_termination_length (min_termination_length_), sample_size(sample_size_), - sampler(sampler_), error (error_) { init(); } + sampler(sampler_), error (error_) { + CV_Assert(min_termination_length_ <= points_size_ && min_termination_length_ >= 0); + if (non_rand_inliers.empty()) + init(); + else non_random_inliers = non_rand_inliers; + } void init () { - // m is sample_size - // N is points_size - + // m is sample_size, N is points_size // non-randomness constraint // The non-randomness requirement prevents PROSAC // from selecting a solution supported by outliers that are @@ -245,20 +231,15 @@ public: // checked ex-post in standard approaches [1]. The distribution // of the cardinalities of sets of random ‘inliers’ is binomial // i-th entry - inlier counts for termination up to i-th point (term length = i+1) - - // ------------------------------------------------------------------------ // initialize the data structures that determine stopping // see probabilities description below. non_random_inliers = std::vector(points_size, 0); - std::vector pn_i_arr(points_size); + std::vector pn_i_arr(points_size, 0); const double beta2compl_beta = beta / (1-beta); const int step_n = 50, max_n = std::min(points_size, 1200); - for (int n = sample_size; n <= points_size; n+=step_n) { - if (n > max_n) { - // skip expensive calculation - break; - } + for (int n = sample_size; n < points_size; n+=step_n) { + if (n > max_n) break; // skip expensive calculation // P^R_n(i) = β^(i−m) (1−β)^(n−i+m) (n−m i−m). (7) i = m,...,N // initial value for i = m = sample_size @@ -288,7 +269,7 @@ public: non_random_inliers[n-1] = i_min; } - // approximate values of binomial distribution + // approximate values of binomial distribution using linear interpolation for (int n = sample_size; n <= points_size; n+=step_n) { if (n-1+step_n >= max_n) { // copy rest of the values @@ -301,19 +282,24 @@ public: non_random_inliers[n+i] = (int)(non_rand_n + (i+1)*step); } } + const std::vector &getNonRandomInliers () const override { return non_random_inliers; } /* * The PROSAC algorithm terminates if the number of inliers I_n* * within the set U_n* satisfies the following conditions: * - * • non-randomness – the probability that I_n* out of n* (termination_length) + * non-randomness – the probability that I_n* out of n* (termination_length) * data points are by chance inliers to an arbitrary incorrect model - * is smaller than Ψ (typically set to 5%) + * is smaller than Sigma (typically set to 5%) * - * • maximality – the probability that a solution with more than + * maximality – the probability that a solution with more than * In* inliers in U_n* exists and was not found after k - * samples is smaller than η0 (typically set to 5%). + * samples is smaller than eta_0 (typically set to 5%). */ - int update (const Mat &model, int inliers_size) override { + int update (const Mat &model, int inliers_size) const override { + int t; return updateTerminationLength(model, inliers_size, t); + } + int updateTerminationLength (const Mat &model, int inliers_size, int &found_termination_length) const override { + found_termination_length = points_size; int predicted_iterations = MAX_ITERATIONS; /* * The termination length n* is chosen to minimize k_n*(η0) subject to I_n* ≥ I_min n*; @@ -327,23 +313,22 @@ public: for (int pt = 0; pt < min_termination_length; pt++) if (errors[pt] < inlier_threshold) num_inliers_under_termination_len++; - for (int termination_len = min_termination_length; termination_len < points_size;termination_len++){ if (errors[termination_len /* = point*/] < inlier_threshold) { num_inliers_under_termination_len++; // non-random constraint must satisfy I_n* ≥ I_min n*. - if (num_inliers_under_termination_len < non_random_inliers[termination_len]) + if (num_inliers_under_termination_len < non_random_inliers[termination_len] || (double) num_inliers_under_termination_len/(points_size) < 0.2) continue; // add 1 to termination length since num_inliers_under_termination_len is updated - const double new_max_samples = log_confidence / log(1 - - std::pow(static_cast(num_inliers_under_termination_len) + const double new_max_samples = log_conf/log(1-pow(static_cast(num_inliers_under_termination_len) / (termination_len+1), sample_size)); if (! std::isinf(new_max_samples) && predicted_iterations > new_max_samples) { predicted_iterations = static_cast(new_max_samples); if (predicted_iterations == 0) break; + found_termination_length = termination_len; if (sampler != nullptr) sampler->setTerminationLength(termination_len); } @@ -352,27 +337,22 @@ public: // compare also when termination length = points_size, // so inliers under termination length is total number of inliers: - const double predicted_iters = log_confidence / log(1 - std::pow + const double predicted_iters = log_conf / log(1 - std::pow (static_cast(inliers_size) / points_size, sample_size)); if (! std::isinf(predicted_iters) && predicted_iters < predicted_iterations) return static_cast(predicted_iters); return predicted_iterations; } - - Ptr clone () const override { - return makePtr(error->clone(), - points_size, sample_size, 1-exp(log_confidence), MAX_ITERATIONS, - min_termination_length, beta, non_randomness_phi, inlier_threshold); - } }; Ptr ProsacTerminationCriteria::create(const Ptr &sampler, const Ptr &error, int points_size_, int sample_size_, double confidence, int max_iterations, - int min_termination_length_, double beta, double non_randomness_phi, double inlier_thresh) { + int min_termination_length_, double beta, double non_randomness_phi, double inlier_thresh, + const std::vector &non_rand_inliers) { return makePtr (sampler, error, points_size_, sample_size_, confidence, max_iterations, min_termination_length_, - beta, non_randomness_phi, inlier_thresh); + beta, non_randomness_phi, inlier_thresh, non_rand_inliers); } }} diff --git a/modules/calib3d/src/usac/utils.cpp b/modules/calib3d/src/usac/utils.cpp index c4d74eb663..5e2206702f 100644 --- a/modules/calib3d/src/usac/utils.cpp +++ b/modules/calib3d/src/usac/utils.cpp @@ -8,9 +8,234 @@ #include namespace cv { namespace usac { -double Utils::getCalibratedThreshold (double threshold, const Matx33d &K1, const Matx33d &K2) { - return threshold / ((K1(0, 0) + K1(1, 1) + - K2(0, 0) + K2(1, 1)) / 4.0); +/* +SolvePoly is used to find only real roots of N-degree polynomial using Sturm sequence. +It recursively finds interval where a root lies, and the actual root is found using Regula-Falsi method. +*/ +class SolvePoly : public SolverPoly { +private: + static int sgn(double val) { + return (double(0) < val) - (val < double(0)); + } + class Poly { + public: + Poly () = default; + Poly (const std::vector &coef_) { + coef = coef_; + checkDegree(); + } + Poly (const Poly &p) { coef = p.coef; } + // a_n x^n + a_n-1 x^(n-1) + ... + a_1 x + a_0 + // coef[i] = a_i + std::vector coef = {0}; + inline int degree() const { return (int)coef.size()-1; } + void multiplyScalar (double s) { + // multiplies polynom by scalar + if (fabs(s) < DBL_EPSILON) { // check if scalar is 0 + coef = {0}; + return; + } + for (double &c : coef) c *= s; + } + void checkDegree() { + int deg = degree(); // approximate degree + // check if coefficients of the highest power is non-zero + while (fabs(coef[deg]) < DBL_EPSILON) { + coef.pop_back(); // remove last zero element + if (--deg == 0) + break; + } + } + double eval (double x) const { + // Horner method a0 + x (a1 + x (a2 + x (a3 + ... + x (an-1 + x an)))) + const int d = degree(); + double y = coef[d]; + for (int i = d; i >= 1; i--) + y = coef[i-1] + x * y; + return y; + } + // at +inf and -inf + std::pair signsAtInf () const { + // lim x->+-inf p(x) = lim x->+-inf a_n x^n + const int d = degree(); + const int s = sgn(coef[d]); // sign of the highest coefficient + // compare even and odd degree + return std::make_pair(s, d % 2 == 0 ? s : -s); + } + Poly derivative () const { + Poly deriv; + if (degree() == 0) + return deriv; + // derive.degree = poly.degree-1; + deriv.coef = std::vector(coef.size()-1); + for (int i = degree(); i > 0; i--) + // (a_n * x^n)' = n * a_n * x^(n-1) + deriv.coef[i-1] = i * coef[i]; + return deriv; + } + void copyFrom (const Poly &p) { coef = p.coef; } + }; + // return remainder + static void dividePoly (const Poly &p1, const Poly &p2, /*Poly "ient,*/ Poly &remainder) { + remainder.copyFrom(p1); + int p2_degree = p2.degree(), remainder_degree = remainder.degree(); + if (p1.degree() < p2_degree) + return; + if (p2_degree == 0) { // special case for dividing polynomial by constant + remainder.multiplyScalar(1/p2.coef[0]); + // quotient.coef[0] = p2.coef[0]; + return; + } + // quotient.coef = std::vector(p1.degree() - p2_degree + 1, 0); + const double p2_term = 1/p2.coef[p2_degree]; + while (remainder_degree >= p2_degree) { + const double temp = remainder.coef[remainder_degree] * p2_term; + // quotient.coef[remainder_degree-p2_degree] = temp; + // polynoms now have the same degree, but p2 is shorter than remainder + for (int i = p2_degree, j = remainder_degree; i >= 0; i--, j--) + remainder.coef[j] -= temp * p2.coef[i]; + remainder.checkDegree(); + remainder_degree = remainder.degree(); + } + } + + constexpr static int REGULA_FALSI_MAX_ITERS = 500, MAX_POWER = 10, MAX_LEVEL = 200; + constexpr static double TOLERANCE = 1e-10, DIFF_TOLERANCE = 1e-7; + + static bool findRootRegulaFalsi (const Poly &poly, double min, double max, double &root) { + double f_min = poly.eval(min), f_max = poly.eval(max); + if (f_min * f_max > 0 || min > max) {// conditions are not fulfilled + return false; + } + int sign = 0, iter = 0; + for (; iter < REGULA_FALSI_MAX_ITERS; iter++) { + root = (f_min * max - f_max * min) / (f_min - f_max); + const double f_root = poly.eval(root); + if (fabs(f_root) < TOLERANCE || fabs(min - max) < DIFF_TOLERANCE) { + return true; // root is found + } + + if (f_root * f_max > 0) { + max = root; f_max = f_root; + if (sign == -1) + f_min *= 0.5; + sign = -1; + } else if (f_min * f_root > 0) { + min = root; f_min = f_root; + if (sign == 1) + f_max *= 0.5; + sign = 1; + } + } + return false; + } + + static int numberOfSignChanges (const std::vector &sturm, double x) { + int prev_sign = 0, sign_changes = 0; + for (const auto &poly : sturm) { + const int s = sgn(poly.eval(x)); + if (s != 0 && prev_sign != 0 && s != prev_sign) + sign_changes++; + prev_sign = s; + } + return sign_changes; + } + + static void findRootsRecursive (const Poly &poly, const std::vector &sturm, double min, double max, + int sign_changes_at_min, int sign_changes_at_max, std::vector &roots, int level) { + const int num_roots = sign_changes_at_min - sign_changes_at_max; + if (level == MAX_LEVEL) { + // roots are too close + const double mid = (min + max) * 0.5; + if (fabs(poly.eval(mid)) < DBL_EPSILON) { + roots.emplace_back(mid); + } + } else if (num_roots == 1) { + double root; + if (findRootRegulaFalsi(poly, min, max, root)) { + roots.emplace_back(root); + } + } else if (num_roots > 1) { // at least 2 roots + const double mid = (min + max) * 0.5; + const int sign_changes_at_mid = numberOfSignChanges(sturm, mid); + // try to split interval equally for the roots + if (sign_changes_at_min - sign_changes_at_mid > 0) + findRootsRecursive(poly, sturm, min, mid, sign_changes_at_min, sign_changes_at_mid, roots, level+1); + if (sign_changes_at_mid - sign_changes_at_max > 0) + findRootsRecursive(poly, sturm, mid, max, sign_changes_at_mid, sign_changes_at_max, roots, level+1); + } + } +public: + int getRealRoots (const std::vector &coeffs, std::vector &real_roots) override { + if (coeffs.empty()) + return 0; + Poly input(coeffs); + if (input.degree() < 1) + return 0; + // derivative of input polynomial + const Poly input_der = input.derivative(); + /////////// build Sturm sequence ////////// + Poly p (input), q (input_der), remainder; + std::vector> signs_at_inf; signs_at_inf.reserve(p.degree()); // +inf, -inf pair + signs_at_inf.emplace_back(p.signsAtInf()); + signs_at_inf.emplace_back(q.signsAtInf()); + std::vector sturm_sequence; sturm_sequence.reserve(input.degree()); + sturm_sequence.emplace_back(input); + sturm_sequence.emplace_back(input_der); + while (q.degree() > 0) { + dividePoly(p, q, remainder); + remainder.multiplyScalar(-1); + p.copyFrom(q); + q.copyFrom(remainder); + sturm_sequence.emplace_back(remainder); + signs_at_inf.emplace_back(remainder.signsAtInf()); + } + ////////// find changes in signs of Sturm sequence ///////// + int num_sign_changes_at_pos_inf = 0, num_sign_changes_at_neg_inf = 0; + int prev_sign_pos_inf = signs_at_inf[0].first, prev_sign_neg_inf = signs_at_inf[0].second; + for (int i = 1; i < (int)signs_at_inf.size(); i++) { + const auto s_pos_inf = signs_at_inf[i].first, s_neg_inf = signs_at_inf[i].second; + // zeros must be ignored + if (s_pos_inf != 0) { + if (prev_sign_pos_inf != 0 && prev_sign_pos_inf != s_pos_inf) + num_sign_changes_at_pos_inf++; + prev_sign_pos_inf = s_pos_inf; + } + if (s_neg_inf != 0) { + if (prev_sign_neg_inf != 0 && prev_sign_neg_inf != s_neg_inf) + num_sign_changes_at_neg_inf++; + prev_sign_neg_inf = s_neg_inf; + } + } + ////////// find roots' bounds for numerical method for roots finding ///////// + double root_neg_bound = -0.01, root_pos_bound = 0.01; + int num_sign_changes_min_x = -1, num_sign_changes_pos_x = -1; // -1 = unknown, trigger next if condition + for (int i = 0; i < MAX_POWER; i++) { + if (num_sign_changes_min_x != num_sign_changes_at_neg_inf) { + root_neg_bound *= 10; + num_sign_changes_min_x = numberOfSignChanges(sturm_sequence, root_neg_bound); + } + if (num_sign_changes_pos_x != num_sign_changes_at_pos_inf) { + root_pos_bound *= 10; + num_sign_changes_pos_x = numberOfSignChanges(sturm_sequence, root_pos_bound); + } + } + /////////// get real roots ////////// + real_roots.clear(); + findRootsRecursive(input, sturm_sequence, root_neg_bound, root_pos_bound, num_sign_changes_min_x, num_sign_changes_pos_x, real_roots, 0 /*level*/); + /////////////////////////////// + if ((int)real_roots.size() > input.degree()) + real_roots.resize(input.degree()); // must not happen, unless some roots repeat + return (int) real_roots.size(); + } +}; +Ptr SolverPoly::create() { + return makePtr(); +} + +double Utils::getCalibratedThreshold (double threshold, const Mat &K1, const Mat &K2) { + const auto * const k1 = (double *) K1.data, * const k2 = (double *) K2.data; + return threshold / ((k1[0] + k1[4] + k2[0] + k2[4]) / 4.0); } /* @@ -20,20 +245,22 @@ double Utils::getCalibratedThreshold (double threshold, const Matx33d &K1, const * 0 k22 k23 * 0 0 1] */ -void Utils::calibratePoints (const Matx33d &K1, const Matx33d &K2, const Mat &points, Mat &calib_points) { - const auto inv1_k11 = float(1 / K1(0, 0)); // 1 / k11 - const auto inv1_k12 = float(-K1(0, 1) / (K1(0, 0)*K1(1, 1))); // -k12 / (k11*k22) - // (-k13*k22 + k12*k23) / (k11*k22) - const auto inv1_k13 = float((-K1(0, 2)*K1(1, 1) + K1(0, 1)*K1(1, 2)) / (K1(0, 0)*K1(1, 1))); - const auto inv1_k22 = float(1 / K1(1, 1)); // 1 / k22 - const auto inv1_k23 = float(-K1(1, 2) / K1(1, 1)); // -k23 / k22 - - const auto inv2_k11 = float(1 / K2(0, 0)); - const auto inv2_k12 = float(-K2(0, 1) / (K2(0, 0)*K2(1, 1))); - const auto inv2_k13 = float((-K2(0, 2)*K2(1, 1) + K2(0, 1)*K2(1, 2)) / (K2(0, 0)*K2(1, 1))); - const auto inv2_k22 = float(1 / K2(1, 1)); - const auto inv2_k23 = float(-K2(1, 2) / K2(1, 1)); +void Utils::calibratePoints (const Mat &K1, const Mat &K2, const Mat &points, Mat &calib_points) { const auto * const points_ = (float *) points.data; + const auto * const k1 = (double *) K1.data; + const auto inv1_k11 = float(1 / k1[0]); // 1 / k11 + const auto inv1_k12 = float(-k1[1] / (k1[0]*k1[4])); // -k12 / (k11*k22) + // (-k13*k22 + k12*k23) / (k11*k22) + const auto inv1_k13 = float((-k1[2]*k1[4] + k1[1]*k1[5]) / (k1[0]*k1[4])); + const auto inv1_k22 = float(1 / k1[4]); // 1 / k22 + const auto inv1_k23 = float(-k1[5] / k1[4]); // -k23 / k22 + + const auto * const k2 = (double *) K2.data; + const auto inv2_k11 = float(1 / k2[0]); + const auto inv2_k12 = float(-k2[1] / (k2[0]*k2[4])); + const auto inv2_k13 = float((-k2[2]*k2[4] + k2[1]*k2[5]) / (k2[0]*k2[4])); + const auto inv2_k22 = float(1 / k2[4]); + const auto inv2_k23 = float(-k2[5] / k2[4]); calib_points = Mat ( points.rows, 4, points.type()); auto * calib_points_ = (float *) calib_points.data; @@ -52,13 +279,15 @@ void Utils::calibratePoints (const Matx33d &K1, const Matx33d &K2, const Mat &po * points is matrix of size |N| x 5, first two columns are image points [u_i, v_i] * calib_norm_pts are K^-1 [u v 1]^T / ||K^-1 [u v 1]^T|| */ -void Utils::calibrateAndNormalizePointsPnP (const Matx33d &K, const Mat &pts, Mat &calib_norm_pts) { +void Utils::calibrateAndNormalizePointsPnP (const Mat &K, const Mat &pts, Mat &calib_norm_pts) { const auto * const points = (float *) pts.data; - const auto inv_k11 = float(1 / K(0, 0)); - const auto inv_k12 = float(-K(0, 1) / (K(0, 0)*K(1, 1))); - const auto inv_k13 = float((-K(0, 2)*K(1, 1) + K(0, 1)*K(1, 2)) / (K(0, 0)*K(1, 1))); - const auto inv_k22 = float(1 / K(1, 1)); - const auto inv_k23 = float(-K(1, 2) / K(1, 1)); + const auto * const k = (double *) K.data; + const auto inv_k11 = float(1 / k[0]); + const auto inv_k12 = float(-k[1] / (k[0]*k[4])); + const auto inv_k13 = float((-k[2]*k[4] + k[1]*k[5]) / (k[0]*k[4])); + const auto inv_k22 = float(1 / k[4]); + const auto inv_k23 = float(-k[5] / k[4]); + calib_norm_pts = Mat (pts.rows, 3, pts.type()); auto * calib_norm_pts_ = (float *) calib_norm_pts.data; @@ -73,9 +302,10 @@ void Utils::calibrateAndNormalizePointsPnP (const Matx33d &K, const Mat &pts, Ma } } -void Utils::normalizeAndDecalibPointsPnP (const Matx33d &K_, Mat &pts, Mat &calib_norm_pts) { - const auto k11 = (float)K_(0, 0), k12 = (float)K_(0, 1), k13 = (float)K_(0, 2), - k22 = (float)K_(1, 1), k23 = (float)K_(1, 2); +void Utils::normalizeAndDecalibPointsPnP (const Mat &K_, Mat &pts, Mat &calib_norm_pts) { + const auto * const K = (double *) K_.data; + const auto k11 = (float)K[0], k12 = (float)K[1], k13 = (float)K[2], + k22 = (float)K[4], k23 = (float)K[5]; calib_norm_pts = Mat (pts.rows, 3, pts.type()); auto * points = (float *) pts.data; auto * calib_norm_pts_ = (float *) calib_norm_pts.data; @@ -98,25 +328,95 @@ void Utils::normalizeAndDecalibPointsPnP (const Matx33d &K_, Mat &pts, Mat &cali * 0 fy ty * 0 0 1] */ -void Utils::decomposeProjection (const Mat &P, Matx33d &K_, Mat &R, Mat &t, bool same_focal) { - const Mat M = P.colRange(0,3); +void Utils::decomposeProjection (const Mat &P, Matx33d &K, Matx33d &R, Vec3d &t, bool same_focal) { + const Matx33d M = P.colRange(0,3); double scale = norm(M.row(2)); scale *= scale; - K_ = Matx33d::eye(); - K_(1,2) = M.row(1).dot(M.row(2)) / scale; - K_(0,2) = M.row(0).dot(M.row(2)) / scale; - K_(1,1) = sqrt(M.row(1).dot(M.row(1)) / scale - K_(1,2)*K_(1,2)); - K_(0,0) = sqrt(M.row(0).dot(M.row(0)) / scale - K_(0,2)*K_(0,2)); + K = Matx33d::eye(); + K(1,2) = M.row(1).dot(M.row(2)) / scale; + K(0,2) = M.row(0).dot(M.row(2)) / scale; + K(1,1) = sqrt(M.row(1).dot(M.row(1)) / scale - K(1,2)*K(1,2)); + K(0,0) = sqrt(M.row(0).dot(M.row(0)) / scale - K(0,2)*K(0,2)); if (same_focal) - K_(0,0) = K_(1,1) = (K_(0,0) + K_(1,1)) / 2; - R = K_.inv() * M / sqrt(scale); + K(0,0) = K(1,1) = (K(0,0) + K(1,1)) / 2; + R = K.inv() * M / sqrt(scale); if (determinant(M) < 0) R *= -1; - t = R * M.inv() * P.col(3); + t = R * M.inv() * Vec3d(P.col(3)); +} + +double Utils::getPoissonCDF (double lambda, int inliers) { + double exp_lamda = exp(-lambda), cdf = exp_lamda, lambda_i_div_fact_i = 1; + for (int i = 1; i <= inliers; i++) { + lambda_i_div_fact_i *= (lambda / i); + cdf += exp_lamda * lambda_i_div_fact_i; + if (fabs(cdf - 1) < DBL_EPSILON) // cdf is almost 1 + break; + } + return cdf; +} + +// since F(E) has rank 2 we use cross product to compute epipole, +// since the third column / row is linearly dependent on two first +// this is faster than SVD +// It is recommended to normalize F, such that ||F|| = 1 +Vec3d Utils::getLeftEpipole (const Mat &F/*E*/) { + Vec3d _e = F.col(0).cross(F.col(2)); // F^T e' = 0; e'^T F = 0 + const auto * const e = _e.val; + if (e[0] <= DBL_EPSILON && e[0] > -DBL_EPSILON && + e[1] <= DBL_EPSILON && e[1] > -DBL_EPSILON && + e[2] <= DBL_EPSILON && e[2] > -DBL_EPSILON) + _e = Vec3d(Mat(F.col(1))).cross(F.col(2)); // if e' is zero + return _e; // e' +} +Vec3d Utils::getRightEpipole (const Mat &F/*E*/) { + Vec3d _e = F.row(0).cross(F.row(2)); // Fe = 0 + const auto * const e = _e.val; + if (e[0] <= DBL_EPSILON && e[0] > -DBL_EPSILON && + e[1] <= DBL_EPSILON && e[1] > -DBL_EPSILON && + e[2] <= DBL_EPSILON && e[2] > -DBL_EPSILON) + _e = F.row(1).cross(F.row(2)); // if e is zero + return _e; +} + +void Utils::densitySort (const Mat &points, int knn, Mat &sorted_points, std::vector &sorted_mask) { + // mask of sorted points (array of indexes) + const int points_size = points.rows, dim = points.cols; + sorted_mask = std::vector(points_size); + for (int i = 0; i < points_size; i++) + sorted_mask[i] = i; + + // get neighbors + FlannNeighborhoodGraph &graph = *FlannNeighborhoodGraph::create(points, points_size, knn, + true /*get distances */, 6, 1); + + std::vector sum_knn_distances (points_size, 0); + for (int p = 0; p < points_size; p++) { + const std::vector &dists = graph.getNeighborsDistances(p); + for (int k = 0; k < knn; k++) + sum_knn_distances[p] += dists[k]; + } + + // compare by sum of distances to k nearest neighbors. + std::sort(sorted_mask.begin(), sorted_mask.end(), [&](int a, int b) { + return sum_knn_distances[a] < sum_knn_distances[b]; + }); + + // copy array of points to array with sorted points + // using @sorted_idx mask of sorted points indexes + + sorted_points = Mat(points_size, dim, points.type()); + const auto * const points_ptr = (float *) points.data; + auto * spoints_ptr = (float *) sorted_points.data; + for (int i = 0; i < points_size; i++) { + const int pt2 = sorted_mask[i] * dim; + for (int j = 0; j < dim; j++) + (*spoints_ptr++) = points_ptr[pt2+j]; + } } Matx33d Math::getSkewSymmetric(const Vec3d &v) { - return Matx33d(0, -v[2], v[1], - v[2], 0, -v[0], - -v[1], v[0], 0); + return {0, -v[2], v[1], + v[2], 0, -v[0], + -v[1], v[0], 0}; } Matx33d Math::rotVec2RotMat (const Vec3d &v) { @@ -124,9 +424,9 @@ Matx33d Math::rotVec2RotMat (const Vec3d &v) { const double x = v[0] / phi, y = v[1] / phi, z = v[2] / phi; const double a = sin(phi), b = cos(phi); // R = I + sin(phi) * skew(v) + (1 - cos(phi) * skew(v)^2 - return Matx33d((b - 1)*y*y + (b - 1)*z*z + 1, -a*z - x*y*(b - 1), a*y - x*z*(b - 1), + return {(b - 1)*y*y + (b - 1)*z*z + 1, -a*z - x*y*(b - 1), a*y - x*z*(b - 1), a*z - x*y*(b - 1), (b - 1)*x*x + (b - 1)*z*z + 1, -a*x - y*z*(b - 1), - -a*y - x*z*(b - 1), a*x - y*z*(b - 1), (b - 1)*x*x + (b - 1)*y*y + 1); + -a*y - x*z*(b - 1), a*x - y*z*(b - 1), (b - 1)*x*x + (b - 1)*y*y + 1}; } Vec3d Math::rotMat2RotVec (const Matx33d &R) { @@ -176,7 +476,7 @@ bool Math::eliminateUpperTriangular (std::vector &a, int m, int n) { // if pivot value is 0 continue if (fabs(pivot) < DBL_EPSILON) - return false; // matrix is not full rank -> terminate + continue; // swap row with maximum pivot value with current row for (int c = r; c < n; c++) @@ -194,6 +494,18 @@ bool Math::eliminateUpperTriangular (std::vector &a, int m, int n) { return true; } +double Utils::intersectionOverUnion (const std::vector &a, const std::vector &b) { + int intersects = 0, unions = 0; + for (int i = 0; i < (int)a.size(); i++) + if (a[i] || b[i]) { + unions++; // one value is true + if (a[i] && b[i]) + intersects++; // a[i] == b[i] and if they both true + } + if (unions == 0) return 0.0; + return (double) intersects / unions; +} + //////////////////////////////////////// RANDOM GENERATOR ///////////////////////////// class UniformRandomGeneratorImpl : public UniformRandomGenerator { private: @@ -209,15 +521,12 @@ public: max_range = max_range_; subset = std::vector(subset_size_); } - int getRandomNumber () override { return rng.uniform(0, max_range); } - int getRandomNumber (int max_rng) override { return rng.uniform(0, max_rng); } - // closed range void resetGenerator (int max_range_) override { CV_CheckGE(0, max_range_, "max range must be greater than 0"); @@ -243,7 +552,6 @@ public: // interval is <0; max_range) void generateUniqueRandomSet (std::vector& sample, int max_range_) override { /* - * necessary condition: * if subset size is bigger than range then array cannot be unique, * so function has infinite loop. */ @@ -282,14 +590,12 @@ public: } return subset; } - void setSubsetSize (int subset_size_) override { + if (subset_size < subset_size_) + subset.resize(subset_size_); subset_size = subset_size_; } int getSubsetSize () const override { return subset_size; } - Ptr clone (int state) const override { - return makePtr(state, max_range, subset_size); - } }; Ptr UniformRandomGenerator::create (int state) { @@ -338,19 +644,17 @@ float Utils::findMedian (std::vector &array) { } else { // even: return average return (quicksort_median(array, length/2 , 0, length-1) + - quicksort_median(array, length/2+1, 0, length-1))/2; + quicksort_median(array, length/2+1, 0, length-1))*.5f; } } -/////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////// Radius Search Graph ///////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////////////////////////////////// class RadiusSearchNeighborhoodGraphImpl : public RadiusSearchNeighborhoodGraph { private: std::vector> graph; public: RadiusSearchNeighborhoodGraphImpl (const Mat &container_, int points_size, - double radius, int flann_search_params, int num_kd_trees) { + double radius, int flann_search_params, int num_kd_trees) { // Radius search OpenCV works only with float data CV_Assert(container_.type() == CV_32F); @@ -363,6 +667,8 @@ public: int pt = 0; for (const auto &n : neighbours) { + if (n.size() <= 1) + continue; auto &graph_row = graph[pt]; graph_row = std::vector(n.size()-1); int j = 0; @@ -373,7 +679,7 @@ public: pt++; } } - + const std::vector> &getGraph () const override { return graph; } inline const std::vector &getNeighbors(int point_idx) const override { return graph[point_idx]; } @@ -384,9 +690,7 @@ Ptr RadiusSearchNeighborhoodGraph::create (const flann_search_params, num_kd_trees); } -/////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////// FLANN Graph ///////////////////////////////////////////// -/////////////////////////////////////////////////////////////////////////////////////////////////// class FlannNeighborhoodGraphImpl : public FlannNeighborhoodGraph { private: std::vector> graph; @@ -425,6 +729,7 @@ public: const std::vector& getNeighborsDistances (int idx) const override { return distances[idx]; } + const std::vector> &getGraph () const override { return graph; } inline const std::vector &getNeighbors(int point_idx) const override { // CV_Assert(point_idx_ < num_vertices); return graph[point_idx]; @@ -438,9 +743,7 @@ Ptr FlannNeighborhoodGraph::create(const Mat &points, k_nearest_neighbors_, get_distances, flann_search_params_, num_kd_trees); } -/////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////// Grid Neighborhood Graph ///////////////////////////////////////// -/////////////////////////////////////////////////////////////////////////////////////////////////// class GridNeighborhoodGraphImpl : public GridNeighborhoodGraph { private: // This struct is used for the nearest neighbors search by griding two images. @@ -460,13 +763,13 @@ private: } }; - std::map> neighbors_map; std::vector> graph; public: GridNeighborhoodGraphImpl (const Mat &container_, int points_size, int cell_size_x_img1, int cell_size_y_img1, int cell_size_x_img2, int cell_size_y_img2, int max_neighbors) { + std::map> neighbors_map; const auto * const container = (float *) container_.data; // -> {neighbors set} // Key is cell position. The value is indexes of neighbors. @@ -490,7 +793,6 @@ public: // store neighbors cells into graph (2D vector) for (const auto &cell : neighbors_map) { const int neighbors_in_cell = static_cast(cell.second.size()); - // only one point in cell -> no neighbors if (neighbors_in_cell < 2) continue; @@ -510,7 +812,7 @@ public: } } } - + const std::vector> &getGraph () const override { return graph; } inline const std::vector &getNeighbors(int point_idx) const override { // Note, neighbors vector also includes point_idx! // return neighbors_map[vertices_to_cells[point_idx]]; @@ -524,4 +826,94 @@ Ptr GridNeighborhoodGraph::create(const Mat &points, return makePtr(points, points_size, cell_size_x_img1_, cell_size_y_img1_, cell_size_x_img2_, cell_size_y_img2_, max_neighbors); } + +class GridNeighborhoodGraph2ImagesImpl : public GridNeighborhoodGraph2Images { +private: + // This struct is used for the nearest neighbors search by griding two images. + struct CellCoord { + int c1x, c1y; + CellCoord (int c1x_, int c1y_) { + c1x = c1x_; c1y = c1y_; + } + bool operator==(const CellCoord &o) const { + return c1x == o.c1x && c1y == o.c1y; + } + bool operator<(const CellCoord &o) const { + if (c1x < o.c1x) return true; + return c1x == o.c1x && c1y < o.c1y; + } + }; + + std::vector> graph; +public: + GridNeighborhoodGraph2ImagesImpl (const Mat &container_, int points_size, + float cell_size_x_img1, float cell_size_y_img1, float cell_size_x_img2, float cell_size_y_img2) { + + std::map> neighbors_map1, neighbors_map2; + const auto * const container = (float *) container_.data; + // Key is cell position. The value is indexes of neighbors. + + const auto cell_sz_x1 = 1.f / cell_size_x_img1, + cell_sz_y1 = 1.f / cell_size_y_img1, + cell_sz_x2 = 1.f / cell_size_x_img2, + cell_sz_y2 = 1.f / cell_size_y_img2; + const int dimension = container_.cols; + for (int i = 0; i < points_size; i++) { + const int idx = dimension * i; + neighbors_map1[CellCoord((int)(container[idx ] * cell_sz_x1), + (int)(container[idx+1] * cell_sz_y1))].emplace_back(i); + neighbors_map2[CellCoord((int)(container[idx+2] * cell_sz_x2), + (int)(container[idx+3] * cell_sz_y2))].emplace_back(i); + } + + //--------- create a graph ---------- + graph = std::vector>(points_size); + + // store neighbors cells into graph (2D vector) + for (const auto &cell : neighbors_map1) { + const int neighbors_in_cell = static_cast(cell.second.size()); + // only one point in cell -> no neighbors + if (neighbors_in_cell < 2) continue; + + const std::vector &neighbors = cell.second; + // ---------- fill graph ----- + // for speed-up we make no symmetric graph, eg, x has a neighbor y, but y does not have x + const int v_in_cell = neighbors[0]; + // there is always at least one neighbor + auto &graph_row = graph[v_in_cell]; + graph_row.reserve(neighbors_in_cell); + for (int n : neighbors) + if (n != v_in_cell) + graph_row.emplace_back(n); + } + + // fill neighbors of a second image + for (const auto &cell : neighbors_map2) { + if (cell.second.size() < 2) continue; + const std::vector &neighbors = cell.second; + const int v_in_cell = neighbors[0]; + auto &graph_row = graph[v_in_cell]; + for (const int &n : neighbors) + if (n != v_in_cell) { + bool has = false; + for (const int &nn : graph_row) + if (n == nn) { + has = true; break; + } + if (!has) graph_row.emplace_back(n); + } + } + } + const std::vector> &getGraph () const override { return graph; } + inline const std::vector &getNeighbors(int point_idx) const override { + // Note, neighbors vector also includes point_idx! + return graph[point_idx]; + } +}; + +Ptr GridNeighborhoodGraph2Images::create(const Mat &points, + int points_size, float cell_size_x_img1_, float cell_size_y_img1_, float cell_size_x_img2_, float cell_size_y_img2_) { + return makePtr(points, points_size, + cell_size_x_img1_, cell_size_y_img1_, cell_size_x_img2_, cell_size_y_img2_); +} }} \ No newline at end of file diff --git a/modules/calib3d/test/test_usac.cpp b/modules/calib3d/test/test_usac.cpp index f7c4b9e4ce..90dcb76ffc 100644 --- a/modules/calib3d/test/test_usac.cpp +++ b/modules/calib3d/test/test_usac.cpp @@ -181,12 +181,9 @@ static double getError (TestSolver test_case, int pt_idx, const cv::Mat &pts1, c if (test_case == TestSolver::Fundam || test_case == TestSolver::Essen) { cv::Mat l2 = model * pt1; cv::Mat l1 = model.t() * pt2; - if (test_case == TestSolver::Fundam) // sampson error - return fabs(pt2.dot(l2)) / sqrt(pow(l1.at(0), 2) + pow(l1.at(1), 2) + - pow(l2.at(0), 2) + pow(l2.at(1), 2)); - else // symmetric geometric distance - return sqrt(pow(pt1.dot(l1),2) / (pow(l1.at(0),2) + pow(l1.at(1),2)) + - pow(pt2.dot(l2),2) / (pow(l2.at(0),2) + pow(l2.at(1),2))); + // Sampson error + return fabs(pt2.dot(l2)) / sqrt(pow(l1.at(0), 2) + pow(l1.at(1), 2) + + pow(l2.at(0), 2) + pow(l2.at(1), 2)); } else if (test_case == TestSolver::PnP) { // PnP, reprojection error cv::Mat img_pt = model * pt2; img_pt /= img_pt.at(2); @@ -340,7 +337,7 @@ TEST_P(usac_Essential, maxiters) { cv::Mat K1 = cv::Mat(cv::Matx33d(1, 0, 0, 0, 1, 0, 0, 0, 1.)); - const double conf = 0.99, thr = 0.5; + const double conf = 0.99, thr = 0.25; int roll_results_sum = 0; for (int iters = 0; iters < 10; iters++) { @@ -361,8 +358,8 @@ TEST_P(usac_Essential, maxiters) { FAIL() << "Essential matrix estimation failed!\n"; else continue; } - EXPECT_NE(roll_results_sum, 0); } + EXPECT_NE(roll_results_sum, 0); } INSTANTIATE_TEST_CASE_P(Calib3d, usac_Essential, UsacMethod::all()); From 024c83646273a763ecdf7e5f66c22f57b5a6046b Mon Sep 17 00:00:00 2001 From: cudawarped <12133430+cudawarped@users.noreply.github.com> Date: Tue, 25 Apr 2023 07:46:02 +0300 Subject: [PATCH 012/105] cv::VideoCapture: change CAP_PROP_FOURCC to prefer codec_id over codec_tag --- modules/videoio/src/cap_ffmpeg_impl.hpp | 63 ++++++++++++------------- modules/videoio/test/test_ffmpeg.cpp | 21 +++++++-- 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index f4d0785f0f..8ba35028c0 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -1707,14 +1707,36 @@ bool CvCapture_FFMPEG::retrieveHWFrame(cv::OutputArray output) #endif } +static inline double getCodecTag(const AVCodecID codec_id) { + const struct AVCodecTag* fallback_tags[] = { + // APIchanges: + // 2012-01-31 - dd6d3b0 - lavf 54.01.0 + // Add avformat_get_riff_video_tags() and avformat_get_riff_audio_tags(). + avformat_get_riff_video_tags(), +#if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(55, 25, 100) && defined LIBAVFORMAT_VERSION_MICRO && LIBAVFORMAT_VERSION_MICRO >= 100 + // APIchanges: ffmpeg only + // 2014-01-19 - 1a193c4 - lavf 55.25.100 - avformat.h + // Add avformat_get_mov_video_tags() and avformat_get_mov_audio_tags(). + avformat_get_mov_video_tags(), +#endif + codec_bmp_tags, // fallback for avformat < 54.1 + NULL }; + return av_codec_get_tag(fallback_tags, codec_id); +} + +static inline double getCodecIdFourcc(const AVCodecID codec_id) +{ + if (codec_id == AV_CODEC_ID_NONE) return -1; + const char* codec_fourcc = _opencv_avcodec_get_name(codec_id); + if (!codec_fourcc || strcmp(codec_fourcc, "unknown_codec") == 0 || strlen(codec_fourcc) != 4) + return getCodecTag(codec_id); + return (double)CV_FOURCC(codec_fourcc[0], codec_fourcc[1], codec_fourcc[2], codec_fourcc[3]); +} + double CvCapture_FFMPEG::getProperty( int property_id ) const { if( !video_st || !context ) return 0; - double codec_tag = 0; - CV_CODEC_ID codec_id = AV_CODEC_ID_NONE; - const char* codec_fourcc = NULL; - switch( property_id ) { case CAP_PROP_POS_MSEC: @@ -1738,34 +1760,11 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const case CAP_PROP_FPS: return get_fps(); case CAP_PROP_FOURCC: { - codec_id = video_st->CV_FFMPEG_CODEC_FIELD->codec_id; - codec_tag = (double) video_st->CV_FFMPEG_CODEC_FIELD->codec_tag; - - if(codec_tag || codec_id == AV_CODEC_ID_NONE) - { - return codec_tag; - } - - codec_fourcc = _opencv_avcodec_get_name(codec_id); - if (!codec_fourcc || strcmp(codec_fourcc, "unknown_codec") == 0 || strlen(codec_fourcc) != 4) - { - const struct AVCodecTag* fallback_tags[] = { - // APIchanges: - // 2012-01-31 - dd6d3b0 - lavf 54.01.0 - // Add avformat_get_riff_video_tags() and avformat_get_riff_audio_tags(). - avformat_get_riff_video_tags(), -#if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(55, 25, 100) && defined LIBAVFORMAT_VERSION_MICRO && LIBAVFORMAT_VERSION_MICRO >= 100 - // APIchanges: ffmpeg only - // 2014-01-19 - 1a193c4 - lavf 55.25.100 - avformat.h - // Add avformat_get_mov_video_tags() and avformat_get_mov_audio_tags(). - avformat_get_mov_video_tags(), -#endif - codec_bmp_tags, // fallback for avformat < 54.1 - NULL }; - return av_codec_get_tag(fallback_tags, codec_id); - } - - return (double) CV_FOURCC(codec_fourcc[0], codec_fourcc[1], codec_fourcc[2], codec_fourcc[3]); + const double fourcc = getCodecIdFourcc(video_st->CV_FFMPEG_CODEC_FIELD->codec_id); + if (fourcc != -1) return fourcc; + const double codec_tag = (double)video_st->CV_FFMPEG_CODEC_FIELD->codec_tag; + if (codec_tag) return codec_tag; + else return -1; } case CAP_PROP_SAR_NUM: return _opencv_ffmpeg_get_sample_aspect_ratio(ic->streams[video_stream]).num; diff --git a/modules/videoio/test/test_ffmpeg.cpp b/modules/videoio/test/test_ffmpeg.cpp index d3f0d5b8fc..e0c2a44075 100644 --- a/modules/videoio/test/test_ffmpeg.cpp +++ b/modules/videoio/test/test_ffmpeg.cpp @@ -226,7 +226,11 @@ const videoio_container_params_t videoio_container_params[] = videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "h264", "h264", "h264", "I420"), videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "h265", "h265", "hevc", "I420"), videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "mjpg.avi", "mjpg", "MJPG", "I420"), +#ifdef _WIN32 // handle old FFmpeg backend - remove when windows shared library is updated videoio_container_params_t(CAP_FFMPEG, "video/sample_322x242_15frames.yuv420p.libx264", "mp4", "h264", "avc1", "I420") +#else + videoio_container_params_t(CAP_FFMPEG, "video/sample_322x242_15frames.yuv420p.libx264", "mp4", "h264", "h264", "I420") +#endif //videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "h264.mkv", "mkv.h264", "h264", "I420"), //videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "h265.mkv", "mkv.h265", "hevc", "I420"), //videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "h264.mp4", "mp4.avc1", "avc1", "I420"), @@ -448,10 +452,21 @@ TEST_P(ffmpeg_get_fourcc, check_short_codecs) const ffmpeg_get_fourcc_param_t ffmpeg_get_fourcc_param[] = { ffmpeg_get_fourcc_param_t("../cv/tracking/faceocc2/data/faceocc2.webm", "VP80"), - ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", "vp09"), - ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libaom-av1.mp4", "av01"), ffmpeg_get_fourcc_param_t("video/big_buck_bunny.h265", "hevc"), - ffmpeg_get_fourcc_param_t("video/big_buck_bunny.h264", "h264") + ffmpeg_get_fourcc_param_t("video/big_buck_bunny.h264", "h264"), +#ifdef _WIN32 // handle old FFmpeg backend - remove when windows shared library is updated + ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", "vp09"), + ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libaom-av1.mp4", "av01") +#else + ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", "VP90"), + ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libaom-av1.mp4", "AV01"), + ffmpeg_get_fourcc_param_t("video/big_buck_bunny.mp4", "FMP4"), + ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.mpeg2video.mp4", "mpg2"), + ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.mjpeg.mp4", "MJPG"), + ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libxvid.mp4", "FMP4"), + ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libx265.mp4", "hevc"), + ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libx264.mp4", "h264") +#endif }; INSTANTIATE_TEST_CASE_P(videoio, ffmpeg_get_fourcc, testing::ValuesIn(ffmpeg_get_fourcc_param)); From 1eaa074a49f40b207bb67c6de46431861106aa86 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 16 Jun 2023 11:28:11 +0200 Subject: [PATCH 013/105] remove line --- modules/videoio/src/cap_images.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/videoio/src/cap_images.cpp b/modules/videoio/src/cap_images.cpp index 21b6d5c0da..18f718d19d 100644 --- a/modules/videoio/src/cap_images.cpp +++ b/modules/videoio/src/cap_images.cpp @@ -258,10 +258,7 @@ std::string icvExtractPattern(const std::string& filename, unsigned *offset) while (pos < len && !isdigit(filename[pos])) pos++; if (pos == len) - { return ""; - CV_Error_(Error::StsBadArg, ("CAP_IMAGES: can't find starting number (in the name of file): %s", filename.c_str())); - } std::string::size_type pos0 = pos; From b6d140236190ae9ecf4574a10ca30e610e1ee483 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Fri, 16 Jun 2023 15:10:47 +0300 Subject: [PATCH 014/105] Fixed barcode to be built without DNN --- .../barcode_decoder/common/super_scale.cpp | 30 ++++++++++++-- .../barcode_decoder/common/super_scale.hpp | 40 +++---------------- 2 files changed, 32 insertions(+), 38 deletions(-) diff --git a/modules/objdetect/src/barcode_decoder/common/super_scale.cpp b/modules/objdetect/src/barcode_decoder/common/super_scale.cpp index 0c9f75f156..4b7099a27d 100644 --- a/modules/objdetect/src/barcode_decoder/common/super_scale.cpp +++ b/modules/objdetect/src/barcode_decoder/common/super_scale.cpp @@ -8,11 +8,14 @@ #include "../../precomp.hpp" #include "super_scale.hpp" - -#ifdef HAVE_OPENCV_DNN +#include "opencv2/core.hpp" +#include "opencv2/core/utils/logger.hpp" namespace cv { namespace barcode { + +#ifdef HAVE_OPENCV_DNN + constexpr static float MAX_SCALE = 4.0f; int SuperScale::init(const std::string &proto_path, const std::string &model_path) @@ -71,7 +74,26 @@ int SuperScale::superResolutionScale(const Mat &src, Mat &dst) } return 0; } + +#else // HAVE_OPENCV_DNN + +int SuperScale::init(const std::string &proto_path, const std::string &model_path) +{ + CV_UNUSED(proto_path); + CV_UNUSED(model_path); + return 0; +} + +void SuperScale::processImageScale(const Mat &src, Mat &dst, float scale, const bool & isEnabled, int sr_max_size) +{ + CV_UNUSED(sr_max_size); + if (isEnabled) + { + CV_LOG_WARNING(NULL, "objdetect/barcode: SuperScaling disabled - OpenCV has been built without DNN support"); + } + resize(src, dst, Size(), scale, scale, INTER_CUBIC); +} +#endif // HAVE_OPENCV_DNN + } // namespace barcode } // namespace cv - -#endif // HAVE_OPENCV_DNN diff --git a/modules/objdetect/src/barcode_decoder/common/super_scale.hpp b/modules/objdetect/src/barcode_decoder/common/super_scale.hpp index 70e47424e4..024ea86e54 100644 --- a/modules/objdetect/src/barcode_decoder/common/super_scale.hpp +++ b/modules/objdetect/src/barcode_decoder/common/super_scale.hpp @@ -9,8 +9,8 @@ #define OPENCV_BARCODE_SUPER_SCALE_HPP #ifdef HAVE_OPENCV_DNN - -#include "opencv2/dnn.hpp" +# include "opencv2/dnn.hpp" +#endif namespace cv { namespace barcode { @@ -26,44 +26,16 @@ public: void processImageScale(const Mat &src, Mat &dst, float scale, const bool &use_sr, int sr_max_size = 160); +#ifdef HAVE_OPENCV_DNN private: dnn::Net srnet_; bool net_loaded_ = false; int superResolutionScale(const cv::Mat &src, cv::Mat &dst); +#endif }; -} // namespace barcode -} // namespace cv - -#else // HAVE_OPENCV_DNN - -#include "opencv2/core.hpp" -#include "opencv2/core/utils/logger.hpp" - -namespace cv { -namespace barcode { - -class SuperScale -{ -public: - int init(const std::string &, const std::string &) - { - return 0; - } - void processImageScale(const Mat &src, Mat &dst, float scale, const bool & isEnabled, int) - { - if (isEnabled) - { - CV_LOG_WARNING(NULL, "objdetect/barcode: SuperScaling disabled - OpenCV has been built without DNN support"); - } - resize(src, dst, Size(), scale, scale, INTER_CUBIC); - } -}; - -} // namespace barcode -} // namespace cv - -#endif // !HAVE_OPENCV_DNN +} // namespace barcode +} // namespace cv #endif // OPENCV_BARCODE_SUPER_SCALE_HPP From ec95efca104f815c65df9257382f9b1cfb2c9728 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Fri, 16 Jun 2023 18:30:21 +0300 Subject: [PATCH 015/105] Merge pull request #23754 from dkurt:remap_linear_transparent Keep inliers for linear remap with BORDER_TRANSPARENT #23754 Address https://github.com/opencv/opencv/issues/23562 ### Pull Request Readiness Checklist resolves https://github.com/opencv/opencv/issues/23562 I do think that this is a bug because with `INTER_CUBIC + BORDER_TRANSPARENT` the last column and row are preserved. So same should be done for `INTER_LINEAR` See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/imgproc/src/imgwarp.cpp | 20 +++++++++++++------- modules/imgproc/test/test_imgwarp.cpp | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index be39419ed1..268555bada 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -757,13 +757,6 @@ static void remapBilinear( const Mat& _src, Mat& _dst, const Mat& _xy, } else { - if( borderType == BORDER_TRANSPARENT && cn != 3 ) - { - D += (X1 - dx)*cn; - dx = X1; - continue; - } - if( cn == 1 ) for( ; dx < X1; dx++, D++ ) { @@ -774,6 +767,12 @@ static void remapBilinear( const Mat& _src, Mat& _dst, const Mat& _xy, { D[0] = cval[0]; } + else if (borderType == BORDER_TRANSPARENT) + { + if (sx < ssize.width && sx >= 0 && + sy < ssize.height && sy >= 0) + D[0] = S0[sy*sstep + sx]; + } else { int sx0, sx1, sy0, sy1; @@ -815,6 +814,13 @@ static void remapBilinear( const Mat& _src, Mat& _dst, const Mat& _xy, for(int k = 0; k < cn; k++ ) D[k] = cval[k]; } + else if (borderType == BORDER_TRANSPARENT) + { + if (sx < ssize.width && sx >= 0 && + sy < ssize.height && sy >= 0) + for(int k = 0; k < cn; k++ ) + D[k] = S0[sy*sstep + sx*cn + k]; + } else { int sx0, sx1, sy0, sy1; diff --git a/modules/imgproc/test/test_imgwarp.cpp b/modules/imgproc/test/test_imgwarp.cpp index c01bccf71e..68208caa04 100644 --- a/modules/imgproc/test/test_imgwarp.cpp +++ b/modules/imgproc/test/test_imgwarp.cpp @@ -1658,5 +1658,21 @@ TEST(Imgproc_warpPolar, identity) #endif } +TEST(Imgproc_Remap, issue_23562) +{ + cv::RNG rng(17); + Mat_ mapx({3, 3}, {0, 1, 2, 0, 1, 2, 0, 1, 2}); + Mat_ mapy({3, 3}, {0, 0, 0, 1, 1, 1, 2, 2, 2}); + for (int cn = 1; cn <= 4; ++cn) { + Mat src(3, 3, CV_32FC(cn)); + rng.fill(src, cv::RNG::UNIFORM, -1, 1); + Mat dst = Mat::zeros(3, 3, CV_32FC(cn)); + Mat ref = src.clone(); + + remap(src, dst, mapx, mapy, INTER_LINEAR, BORDER_TRANSPARENT); + ASSERT_EQ(0.0, cvtest::norm(ref, dst, NORM_INF)) << "channels=" << cn; + } +} + }} // namespace /* End of file. */ From 433c36445691c081c75bb9cefbacd7f9384334dc Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Fri, 16 Jun 2023 19:58:20 +0300 Subject: [PATCH 016/105] Merge pull request #23724 from dkurt:java_without_ant Build Java without ANT #23724 ### Pull Request Readiness Checklist Enables a path of building Java bindings without ANT * Able to build OpenCV JAR and Docs without ANT ``` -- Java: -- ant: NO -- JNI: /usr/lib/jvm/default-java/include /usr/lib/jvm/default-java/include/linux /usr/lib/jvm/default-java/include -- Java wrappers: YES -- Java tests: NO ``` * Possible to build OpenCV JAR without ANT but tests still require ANT **Merge with**: https://github.com/opencv/opencv_contrib/pull/3502 Notes: - Use `OPENCV_JAVA_IGNORE_ANT=1` to force "Java" flow for building Java bindings - Java tests still require Apache ANT - JAR doesn't include `.java` source code files. See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- CMakeLists.txt | 12 +- modules/core/misc/java/src/java/core+Mat.java | 3 + modules/dnn/include/opencv2/dnn/dnn.hpp | 1 - modules/java/CMakeLists.txt | 4 +- modules/java/jar/CMakeLists.txt | 117 +++++++++++++----- modules/java/jar/MANIFEST.MF.in | 5 + modules/java/jar/list_java_sources.cmake | 19 +++ 7 files changed, 130 insertions(+), 31 deletions(-) create mode 100644 modules/java/jar/MANIFEST.MF.in create mode 100644 modules/java/jar/list_java_sources.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index abff989606..ceeb8b8d7d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -777,6 +777,15 @@ if(BUILD_JAVA) include(cmake/android/OpenCVDetectAndroidSDK.cmake) else() include(cmake/OpenCVDetectApacheAnt.cmake) + if(ANT_EXECUTABLE AND NOT OPENCV_JAVA_IGNORE_ANT) + ocv_update(OPENCV_JAVA_SDK_BUILD_TYPE "ANT") + elseif(NOT ANDROID) + find_package(Java) + if(Java_FOUND) + include(UseJava) + ocv_update(OPENCV_JAVA_SDK_BUILD_TYPE "JAVA") + endif() + endif() find_package(JNI) endif() endif() @@ -1802,9 +1811,10 @@ if(BUILD_JAVA) status(" Java:" BUILD_FAT_JAVA_LIB THEN "export all functions" ELSE "") status(" ant:" ANT_EXECUTABLE THEN "${ANT_EXECUTABLE} (ver ${ANT_VERSION})" ELSE NO) if(NOT ANDROID) + status(" Java:" Java_FOUND THEN "YES (ver ${Java_VERSION})" ELSE NO) status(" JNI:" JNI_INCLUDE_DIRS THEN "${JNI_INCLUDE_DIRS}" ELSE NO) endif() - status(" Java wrappers:" HAVE_opencv_java THEN YES ELSE NO) + status(" Java wrappers:" HAVE_opencv_java THEN "YES (${OPENCV_JAVA_SDK_BUILD_TYPE})" ELSE NO) status(" Java tests:" BUILD_TESTS AND opencv_test_java_BINARY_DIR THEN YES ELSE NO) endif() diff --git a/modules/core/misc/java/src/java/core+Mat.java b/modules/core/misc/java/src/java/core+Mat.java index 2d9ee314a3..8e98284dde 100644 --- a/modules/core/misc/java/src/java/core+Mat.java +++ b/modules/core/misc/java/src/java/core+Mat.java @@ -470,6 +470,7 @@ public class Mat { * Element-wise multiplication with scale factor * @param m operand with with which to perform element-wise multiplication * @param scale scale factor + * @return reference to a new Mat object */ public Mat mul(Mat m, double scale) { return new Mat(n_mul(nativeObj, m.nativeObj, scale)); @@ -478,6 +479,7 @@ public class Mat { /** * Element-wise multiplication * @param m operand with with which to perform element-wise multiplication + * @return reference to a new Mat object */ public Mat mul(Mat m) { return new Mat(n_mul(nativeObj, m.nativeObj)); @@ -487,6 +489,7 @@ public class Mat { * Matrix multiplication * @param m operand with with which to perform matrix multiplication * @see Core#gemm(Mat, Mat, double, Mat, double, Mat, int) + * @return reference to a new Mat object */ public Mat matMul(Mat m) { return new Mat(n_matMul(nativeObj, m.nativeObj)); diff --git a/modules/dnn/include/opencv2/dnn/dnn.hpp b/modules/dnn/include/opencv2/dnn/dnn.hpp index 60f68fce6e..d61f7191bc 100644 --- a/modules/dnn/include/opencv2/dnn/dnn.hpp +++ b/modules/dnn/include/opencv2/dnn/dnn.hpp @@ -894,7 +894,6 @@ CV__DNN_INLINE_NS_BEGIN * @param cfgFile path to the .cfg file with text description of the network architecture. * @param darknetModel path to the .weights file with learned network. * @returns Network object that ready to do forward, throw an exception in failure cases. - * @returns Net object. */ CV_EXPORTS_W Net readNetFromDarknet(const String &cfgFile, const String &darknetModel = String()); diff --git a/modules/java/CMakeLists.txt b/modules/java/CMakeLists.txt index 33eca05988..7fe90a0cb3 100644 --- a/modules/java/CMakeLists.txt +++ b/modules/java/CMakeLists.txt @@ -3,7 +3,9 @@ if(OPENCV_INITIAL_PASS) add_subdirectory(generator) endif() -if(APPLE_FRAMEWORK OR WINRT OR NOT PYTHON_DEFAULT_AVAILABLE OR NOT (ANT_EXECUTABLE OR ANDROID_PROJECTS_BUILD_TYPE STREQUAL "GRADLE") +if(APPLE_FRAMEWORK OR WINRT + OR NOT PYTHON_DEFAULT_AVAILABLE + OR NOT (ANT_EXECUTABLE OR Java_FOUND OR ANDROID_PROJECTS_BUILD_TYPE STREQUAL "GRADLE") OR NOT (JNI_FOUND OR (ANDROID AND (NOT DEFINED ANDROID_NATIVE_API_LEVEL OR ANDROID_NATIVE_API_LEVEL GREATER 7))) OR BUILD_opencv_world ) diff --git a/modules/java/jar/CMakeLists.txt b/modules/java/jar/CMakeLists.txt index 33817bcc62..97533d0ee4 100644 --- a/modules/java/jar/CMakeLists.txt +++ b/modules/java/jar/CMakeLists.txt @@ -5,12 +5,13 @@ set(OPENCV_JAVA_DIR "${CMAKE_CURRENT_BINARY_DIR}/opencv" CACHE INTERNAL "") file(REMOVE_RECURSE "${OPENCV_JAVA_DIR}") file(REMOVE "${OPENCV_DEPHELPER}/${the_module}_jar_source_copy") -file(MAKE_DIRECTORY "${OPENCV_JAVA_DIR}/build/classes") set(java_src_dir "${OPENCV_JAVA_DIR}/java") file(MAKE_DIRECTORY "${java_src_dir}") -set(JAR_NAME opencv-${OPENCV_JAVA_LIB_NAME_SUFFIX}.jar) -set(OPENCV_JAR_FILE "${OpenCV_BINARY_DIR}/bin/${JAR_NAME}" CACHE INTERNAL "") +set(JAR_NAME_WE opencv-${OPENCV_JAVA_LIB_NAME_SUFFIX}) +set(JAR_NAME ${JAR_NAME_WE}.jar) +set(OPENCV_JAR_DIR "${OpenCV_BINARY_DIR}/bin/" CACHE INTERNAL "") +set(OPENCV_JAR_FILE "${OPENCV_JAR_DIR}${JAR_NAME}" CACHE INTERNAL "") ocv_copyfiles_append_dir(JAVA_SRC_COPY "${OPENCV_JAVA_BINDINGS_DIR}/gen/java" "${java_src_dir}") @@ -34,38 +35,98 @@ if(OPENCV_JAVADOC_LINK_URL) set(CMAKE_CONFIG_OPENCV_JAVADOC_LINK "link=\"${OPENCV_JAVADOC_LINK_URL}\"") endif() -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/build.xml.in" "${OPENCV_JAVA_DIR}/build.xml" @ONLY) -list(APPEND depends "${OPENCV_JAVA_DIR}/build.xml") +if(OPENCV_JAVA_SDK_BUILD_TYPE STREQUAL "ANT") + file(MAKE_DIRECTORY "${OPENCV_JAVA_DIR}/build/classes") -ocv_cmake_byproducts(__byproducts BYPRODUCTS "${OPENCV_JAR_FILE}") -add_custom_command(OUTPUT "${OPENCV_DEPHELPER}/${the_module}_jar" - ${__byproducts} # required for add_custom_target() by ninja - COMMAND ${ANT_EXECUTABLE} -noinput -k jar - COMMAND ${CMAKE_COMMAND} -E touch "${OPENCV_DEPHELPER}/${the_module}_jar" - WORKING_DIRECTORY "${OPENCV_JAVA_DIR}" - DEPENDS ${depends} - COMMENT "Generating ${JAR_NAME}" -) -add_custom_target(${the_module}_jar DEPENDS "${OPENCV_DEPHELPER}/${the_module}_jar") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/build.xml.in" "${OPENCV_JAVA_DIR}/build.xml" @ONLY) + list(APPEND depends "${OPENCV_JAVA_DIR}/build.xml") + + ocv_cmake_byproducts(__byproducts BYPRODUCTS "${OPENCV_JAR_FILE}") + add_custom_command(OUTPUT "${OPENCV_DEPHELPER}/${the_module}_jar" + ${__byproducts} # required for add_custom_target() by ninja + COMMAND ${ANT_EXECUTABLE} -noinput -k jar + COMMAND ${CMAKE_COMMAND} -E touch "${OPENCV_DEPHELPER}/${the_module}_jar" + WORKING_DIRECTORY "${OPENCV_JAVA_DIR}" + DEPENDS ${depends} + COMMENT "Generating ${JAR_NAME}" + ) + add_custom_target(${the_module}_jar DEPENDS "${OPENCV_DEPHELPER}/${the_module}_jar") +elseif(OPENCV_JAVA_SDK_BUILD_TYPE STREQUAL "JAVA") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/MANIFEST.MF.in" "${OPENCV_JAVA_DIR}/MANIFEST.MF" @ONLY) + list(APPEND depends "${OPENCV_JAVA_DIR}/MANIFEST.MF") + + ocv_cmake_byproducts(__byproducts BYPRODUCTS "${OPENCV_JAVA_DIR}/java_sources") + add_custom_command(OUTPUT "${OPENCV_DEPHELPER}/${the_module}_jar" + BYPRODUCTS ${__byproducts} # required for add_custom_target() by ninja + DEPENDS ${depends} + COMMAND ${CMAKE_COMMAND} -E touch "${OPENCV_DEPHELPER}/${the_module}_jar" + COMMAND ${CMAKE_COMMAND} + -D OPENCV_JAVA_DIR="${OPENCV_JAVA_DIR}/java" + -D OUTPUT="${OPENCV_JAVA_DIR}/java_sources" + -P "${CMAKE_CURRENT_SOURCE_DIR}/list_java_sources.cmake" + ) + + add_custom_target(${the_module}_jar_sources + DEPENDS "${OPENCV_DEPHELPER}/${the_module}_jar" + ) + + list(APPEND CMAKE_JAVA_COMPILE_FLAGS -encoding utf-8 ${OPENCV_EXTRA_JAVA_COMPILE_FLAGS}) + + add_jar(${the_module}_jar + SOURCES "@${OPENCV_JAVA_DIR}/java_sources" + MANIFEST "${OPENCV_JAVA_DIR}/MANIFEST.MF" + OUTPUT_NAME "${JAR_NAME_WE}" + OUTPUT_DIR "${OPENCV_JAR_DIR}") + + add_dependencies(${the_module}_jar ${the_module}_jar_sources) +else() + ocv_assert(0) +endif() install(FILES ${OPENCV_JAR_FILE} OPTIONAL DESTINATION ${OPENCV_JAR_INSTALL_PATH} COMPONENT java) add_dependencies(${the_module} ${the_module}_jar) if(BUILD_DOCS) - add_custom_command(OUTPUT "${OPENCV_DEPHELPER}/${the_module}doc" - COMMAND ${ANT_EXECUTABLE} -noinput -k javadoc - COMMAND ${CMAKE_COMMAND} -E touch "${OPENCV_DEPHELPER}/${the_module}doc" - WORKING_DIRECTORY "${OPENCV_JAVA_DIR}" - DEPENDS ${depends} - COMMENT "Generating Javadoc" - ) - add_custom_target(${the_module}doc DEPENDS "${OPENCV_DEPHELPER}/${the_module}doc") - install(DIRECTORY ${OpenCV_BINARY_DIR}/doc/doxygen/html/javadoc - DESTINATION "${OPENCV_DOC_INSTALL_PATH}/html" - COMPONENT "docs" OPTIONAL - ${compatible_MESSAGE_NEVER} - ) + if(OPENCV_JAVA_SDK_BUILD_TYPE STREQUAL "ANT") + add_custom_command(OUTPUT "${OPENCV_DEPHELPER}/${the_module}doc" + COMMAND ${ANT_EXECUTABLE} -noinput -k javadoc + COMMAND ${CMAKE_COMMAND} -E touch "${OPENCV_DEPHELPER}/${the_module}doc" + WORKING_DIRECTORY "${OPENCV_JAVA_DIR}" + DEPENDS ${depends} + COMMENT "Generating Javadoc" + ) + add_custom_target(${the_module}doc DEPENDS "${OPENCV_DEPHELPER}/${the_module}doc") + + install(DIRECTORY ${OpenCV_BINARY_DIR}/doc/doxygen/html/javadoc + DESTINATION "${OPENCV_DOC_INSTALL_PATH}/html" + COMPONENT "docs" OPTIONAL + ${compatible_MESSAGE_NEVER} + ) + elseif(OPENCV_JAVA_SDK_BUILD_TYPE STREQUAL "JAVA") + set(Java_JAVADOC_EXECUTABLE ${Java_JAVADOC_EXECUTABLE} -encoding utf-8) + + # create_javadoc produces target ${_target}_javadoc + create_javadoc(${the_module} + FILES "@${OPENCV_JAVA_DIR}/java_sources" + SOURCEPATH "${OPENCV_JAVA_DIR}/java" + INSTALLPATH "${OPENCV_JAVADOC_DESTINATION}" + WINDOWTITLE "OpenCV ${OPENCV_VERSION_PLAIN} Java documentation" + DOCTITLE "OpenCV Java documentation (${OPENCV_VERSION})" + VERSION TRUE + ) + add_dependencies(${the_module}_javadoc ${the_module}_jar_sources) + add_custom_target(${the_module}doc DEPENDS ${the_module}_javadoc) + + install(DIRECTORY ${OpenCV_BINARY_DIR}/doc/doxygen/html/javadoc/${the_module}/ + DESTINATION "${OPENCV_DOC_INSTALL_PATH}/html/javadoc" + COMPONENT "docs" OPTIONAL + ${compatible_MESSAGE_NEVER} + ) + else() + ocv_assert(0) + endif() + set(CMAKE_DOXYGEN_JAVADOC_NODE "" CACHE INTERNAL "Link to the Java documentation") # set to the cache to make it global diff --git a/modules/java/jar/MANIFEST.MF.in b/modules/java/jar/MANIFEST.MF.in new file mode 100644 index 0000000000..07d2fda79b --- /dev/null +++ b/modules/java/jar/MANIFEST.MF.in @@ -0,0 +1,5 @@ +Specification-Title: OpenCV +Specification-Version: @OPENCV_VERSION@ +Implementation-Title: OpenCV +Implementation-Version: @OPENCV_VCSVERSION@ +Implementation-Date: @OPENCV_TIMESTAMP@ diff --git a/modules/java/jar/list_java_sources.cmake b/modules/java/jar/list_java_sources.cmake new file mode 100644 index 0000000000..e39a630e18 --- /dev/null +++ b/modules/java/jar/list_java_sources.cmake @@ -0,0 +1,19 @@ +file(GLOB_RECURSE java_sources "${OPENCV_JAVA_DIR}/*.java") + +set(__sources "") + +foreach(dst ${java_sources}) + set(__sources "${__sources}${dst}\n") +endforeach() + +function(ocv_update_file filepath content) + if(EXISTS "${filepath}") + file(READ "${filepath}" actual_content) + else() + set(actual_content "") + endif() + if(NOT ("${actual_content}" STREQUAL "${content}")) + file(WRITE "${filepath}" "${content}") + endif() +endfunction() +ocv_update_file("${OUTPUT}" "${__sources}") From 94703fc5b082a7391a4bd4bd1f872d872ffc55fe Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Fri, 16 Jun 2023 20:03:19 +0300 Subject: [PATCH 017/105] Merge pull request #23816 from VadimLevin:dev/vlevin/export-all-caps-enum-constants Export enums ALL_CAPS version to typing stub files #23816 - Export ALL_CAPS versions alongside from normal names for enum constants, since both versions are available in runtime - Change enum names entries comments to documentary strings Before patch ```python RMat_Access_R: int RMat_Access_W: int RMat_Access = int # One of [R, W] ``` After patch ```python RMat_Access_R: int RMAT_ACCESS_R: int RMat_Access_W: int RMAT_ACCESS_W: int RMat_Access = int """One of [RMat_Access_R, RMAT_ACCESS_R, RMat_Access_W, RMAT_ACCESS_W]""" ``` Resolves: #23776 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../typing_stubs_generation/generation.py | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index 7f3380ae89..b830408b26 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -2,8 +2,9 @@ __all__ = ("generate_typing_stubs", ) from io import StringIO from pathlib import Path +import re from typing import (Generator, Type, Callable, NamedTuple, Union, Set, Dict, - Collection) + Collection, Tuple, List) import warnings from .ast_utils import get_enclosing_namespace, get_enum_module_and_export_name @@ -272,7 +273,8 @@ def _generate_class_stub(class_node: ClassNode, output_stream: StringIO, def _generate_constant_stub(constant_node: ConstantNode, output_stream: StringIO, indent: int = 0, - extra_export_prefix: str = "") -> None: + extra_export_prefix: str = "", + generate_uppercase_version: bool = True) -> Tuple[str, ...]: """Generates stub for the provided constant node. Args: @@ -280,18 +282,32 @@ def _generate_constant_stub(constant_node: ConstantNode, output_stream (StringIO): Output stream for constant stub. indent (int, optional): Indent used for each line written to `output_stream`. Defaults to 0. - extra_export_prefix (str, optional) Extra prefix added to the export + extra_export_prefix (str, optional): Extra prefix added to the export constant name. Defaults to empty string. + generate_uppercase_version (bool, optional): Generate uppercase version + alongside the normal one. Defaults to True. + + Returns: + Tuple[str, ...]: exported constants names. """ - output_stream.write( - "{indent}{prefix}{name}: {value_type}\n".format( - prefix=extra_export_prefix, - name=constant_node.export_name, - value_type=constant_node.value_type, - indent=" " * indent + def write_constant_to_stream(export_name: str) -> None: + output_stream.write( + "{indent}{name}: {value_type}\n".format( + name=export_name, + value_type=constant_node.value_type, + indent=" " * indent + ) ) - ) + + export_name = extra_export_prefix + constant_node.export_name + write_constant_to_stream(export_name) + if generate_uppercase_version: + uppercase_name = re.sub(r"([a-z])([A-Z])", r"\1_\2", export_name).upper() + if export_name != uppercase_name: + write_constant_to_stream(uppercase_name) + return export_name, uppercase_name + return export_name, def _generate_enumeration_stub(enumeration_node: EnumerationNode, @@ -356,18 +372,20 @@ def _generate_enumeration_stub(enumeration_node: EnumerationNode, entries_extra_prefix = extra_export_prefix if enumeration_node.is_scoped: entries_extra_prefix += enumeration_node.export_name + "_" + generated_constants_entries: List[str] = [] for entry in enumeration_node.constants.values(): - _generate_constant_stub(entry, output_stream, indent, entries_extra_prefix) + generated_constants_entries.extend( + _generate_constant_stub(entry, output_stream, indent, entries_extra_prefix) + ) # Unnamed enumerations are skipped as definition if enumeration_node.export_name.endswith(""): output_stream.write("\n") return output_stream.write( - "{indent}{export_prefix}{name} = int # One of [{entries}]\n\n".format( + '{indent}{export_prefix}{name} = int\n{indent}"""One of [{entries}]"""\n\n'.format( export_prefix=extra_export_prefix, name=enumeration_node.export_name, - entries=", ".join(entry.export_name - for entry in enumeration_node.constants.values()), + entries=", ".join(generated_constants_entries), indent=" " * indent ) ) From a4a739b99e8b6df2ec6de750c459e56e85039f28 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Fri, 16 Jun 2023 21:51:17 +0300 Subject: [PATCH 018/105] Force mat_wrapper import to satisfy dependencies for MatLike alias. --- modules/python/src2/typing_stubs_generation/generation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index b830408b26..5b3a2ebe00 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -759,6 +759,8 @@ def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None: output_stream.write(f' "{alias_name}",\n') output_stream.write("]\n\n") + # HACK: force add cv2.mat_wrapper import to handle MatLike alias + required_imports.add("import cv2.mat_wrapper") _write_required_imports(required_imports, output_stream) # Add type checking time definitions as generated __init__.py content From d6d15c136ac235532250ac9e2b7e61541ddcc067 Mon Sep 17 00:00:00 2001 From: Ulvi YELEN Date: Sat, 17 Jun 2023 09:38:57 +0300 Subject: [PATCH 019/105] if browser supports wasm but only asm.js path provided use asm.js as fallback --- modules/js/src/loader.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/js/src/loader.js b/modules/js/src/loader.js index ea100e8601..1690e0468d 100644 --- a/modules/js/src/loader.js +++ b/modules/js/src/loader.js @@ -73,6 +73,11 @@ async function loadOpenCV(paths, onloadCallback) { console.log("The OpenCV.js for wasm is loaded now"); } else if (wasmSupported) { console.log("The browser supports wasm, but the path of OpenCV.js for wasm is empty"); + + if (asmPath != "") { + OPENCV_URL = asmPath; + console.log("The OpenCV.js for Asm.js is loaded as fallback."); + } } if (OPENCV_URL === "") { From ddcbd2cc268f6293bb636a41c1aa19dfb3c181ad Mon Sep 17 00:00:00 2001 From: lamm45 <96844552+lamm45@users.noreply.github.com> Date: Mon, 19 Jun 2023 08:11:01 -0400 Subject: [PATCH 020/105] Merge pull request #22798 from lamm45:distransform-large Fix distransform to work with large images #22798 This attempts to fix the following bug which was caused by storing squares of large integers into 32-bit floating point variables: https://github.com/opencv/opencv/issues/22732 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/imgproc/src/distransform.cpp | 31 +++++++------- .../imgproc/test/test_distancetransform.cpp | 42 +++++++++++++++++++ 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/modules/imgproc/src/distransform.cpp b/modules/imgproc/src/distransform.cpp index a2c5a0d5fd..dd22d4f50b 100644 --- a/modules/imgproc/src/distransform.cpp +++ b/modules/imgproc/src/distransform.cpp @@ -448,7 +448,7 @@ static void getDistanceTransformMask( int maskType, float *metrics ) struct DTColumnInvoker : ParallelLoopBody { - DTColumnInvoker( const Mat* _src, Mat* _dst, const int* _sat_tab, const float* _sqr_tab) + DTColumnInvoker( const Mat* _src, Mat* _dst, const int* _sat_tab, const int* _sqr_tab) { src = _src; dst = _dst; @@ -481,7 +481,7 @@ struct DTColumnInvoker : ParallelLoopBody { dist = dist + 1 - sat_tab[dist - d[j]]; d[j] = dist; - dptr[0] = sqr_tab[dist]; + dptr[0] = (float)sqr_tab[dist]; } } } @@ -489,12 +489,12 @@ struct DTColumnInvoker : ParallelLoopBody const Mat* src; Mat* dst; const int* sat_tab; - const float* sqr_tab; + const int* sqr_tab; }; struct DTRowInvoker : ParallelLoopBody { - DTRowInvoker( Mat* _dst, const float* _sqr_tab, const float* _inv_tab ) + DTRowInvoker( Mat* _dst, const int* _sqr_tab, const float* _inv_tab ) { dst = _dst; sqr_tab = _sqr_tab; @@ -529,7 +529,7 @@ struct DTRowInvoker : ParallelLoopBody for(;;k--) { p = v[k]; - float s = (fq + sqr_tab[q] - d[p] - sqr_tab[p])*inv_tab[q - p]; + float s = (fq - d[p] + (sqr_tab[q]-sqr_tab[p]))*inv_tab[q - p]; if( s > z[k] ) { k++; @@ -552,28 +552,28 @@ struct DTRowInvoker : ParallelLoopBody } Mat* dst; - const float* sqr_tab; + const int* sqr_tab; const float* inv_tab; }; static void trueDistTrans( const Mat& src, Mat& dst ) { - const float inf = 1e15f; + const int inf = INT_MAX; CV_Assert( src.size() == dst.size() ); CV_Assert( src.type() == CV_8UC1 && dst.type() == CV_32FC1 ); int i, m = src.rows, n = src.cols; - cv::AutoBuffer _buf(std::max(m*2*sizeof(float) + (m*3+1)*sizeof(int), n*2*sizeof(float))); + cv::AutoBuffer _buf(std::max(m*2*sizeof(int) + (m*3+1)*sizeof(int), n*2*sizeof(float))); // stage 1: compute 1d distance transform of each column - float* sqr_tab = (float*)_buf.data(); + int* sqr_tab = (int*)_buf.data(); int* sat_tab = cv::alignPtr((int*)(sqr_tab + m*2), sizeof(int)); int shift = m*2; for( i = 0; i < m; i++ ) - sqr_tab[i] = (float)(i*i); + sqr_tab[i] = i*i; for( i = m; i < m*2; i++ ) sqr_tab[i] = inf; for( i = 0; i < shift; i++ ) @@ -584,13 +584,14 @@ trueDistTrans( const Mat& src, Mat& dst ) cv::parallel_for_(cv::Range(0, n), cv::DTColumnInvoker(&src, &dst, sat_tab, sqr_tab), src.total()/(double)(1<<16)); // stage 2: compute modified distance transform for each row - float* inv_tab = sqr_tab + n; + float* inv_tab = (float*)sqr_tab + n; - inv_tab[0] = sqr_tab[0] = 0.f; + inv_tab[0] = 0.f; + sqr_tab[0] = 0; for( i = 1; i < n; i++ ) { inv_tab[i] = (float)(0.5/i); - sqr_tab[i] = (float)(i*i); + sqr_tab[i] = i*i; } cv::parallel_for_(cv::Range(0, m), cv::DTRowInvoker(&dst, sqr_tab, inv_tab)); @@ -752,7 +753,9 @@ void cv::distanceTransform( InputArray _src, OutputArray _dst, OutputArray _labe CV_IPP_CHECK() { #if IPP_DISABLE_PERF_TRUE_DIST_MT - if(cv::getNumThreads()<=1 || (src.total()<(int)(1<<14))) + // IPP uses floats, but 4097 cannot be squared into a float + if((cv::getNumThreads()<=1 || (src.total()<(int)(1<<14))) && + src.rows < 4097 && src.cols < 4097) #endif { IppStatus status; diff --git a/modules/imgproc/test/test_distancetransform.cpp b/modules/imgproc/test/test_distancetransform.cpp index 32fc7aa2c3..742595631a 100644 --- a/modules/imgproc/test/test_distancetransform.cpp +++ b/modules/imgproc/test/test_distancetransform.cpp @@ -302,4 +302,46 @@ BIGDATA_TEST(Imgproc_DistanceTransform, large_image_12218) EXPECT_EQ(nz, (size.height*size.width / 2)); } +TEST(Imgproc_DistanceTransform, wide_image_22732) +{ + Mat src = Mat::zeros(1, 4099, CV_8U); // 4099 or larger used to be bad + Mat dist(src.rows, src.cols, CV_32F); + distanceTransform(src, dist, DIST_L2, DIST_MASK_PRECISE, CV_32F); + int nz = countNonZero(dist); + EXPECT_EQ(nz, 0); +} + +TEST(Imgproc_DistanceTransform, large_square_22732) +{ + Mat src = Mat::zeros(8000, 8005, CV_8U), dist; + distanceTransform(src, dist, DIST_L2, DIST_MASK_PRECISE, CV_32F); + int nz = countNonZero(dist); + EXPECT_EQ(dist.size(), src.size()); + EXPECT_EQ(dist.type(), CV_32F); + EXPECT_EQ(nz, 0); + + Point p0(src.cols-1, src.rows-1); + src.setTo(1); + src.at(p0) = 0; + distanceTransform(src, dist, DIST_L2, DIST_MASK_PRECISE, CV_32F); + EXPECT_EQ(dist.size(), src.size()); + EXPECT_EQ(dist.type(), CV_32F); + bool first = true; + int nerrs = 0; + for (int y = 0; y < dist.rows; y++) + for (int x = 0; x < dist.cols; x++) { + float d = dist.at(y, x); + double dx = (double)(x - p0.x), dy = (double)(y - p0.y); + float d0 = (float)sqrt(dx*dx + dy*dy); + if (std::abs(d0 - d) > 1) { + if (first) { + printf("y=%d, x=%d. dist_ref=%.2f, dist=%.2f\n", y, x, d0, d); + first = false; + } + nerrs++; + } + } + EXPECT_EQ(0, nerrs) << "reference distance map is different from computed one at " << nerrs << " pixels\n"; +} + }} // namespace From 0cf45b89ec8e71ac1aa2c95a4ffda0cfcbf8ae62 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Tue, 20 Jun 2023 10:33:08 +0100 Subject: [PATCH 021/105] Merge pull request #23796 from TolyaTalamanov:at/align-ie-backend-with-latest-openvino G-API: Align IE Backend with the latest OpenVINO version #23796 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/gapi/src/backends/ie/giebackend.cpp | 33 ++++- .../gapi/test/infer/gapi_infer_ie_test.cpp | 133 +++++++++++++++++- 2 files changed, 157 insertions(+), 9 deletions(-) diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index a257c21252..6026f29ae5 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -949,7 +949,11 @@ inline IE::Blob::Ptr extractBlob(IECallContext& ctx, auto y_blob = ctx.uu.rctx->CreateBlob(blob_params->first.first, blob_params->first.second); auto uv_blob = ctx.uu.rctx->CreateBlob(blob_params->second.first, blob_params->second.second); -#if INF_ENGINE_RELEASE >= 2021010000 +#if INF_ENGINE_RELEASE > 2023000000 + cv::util::throw_error(std::logic_error( + "IE Backend: NV12 feature has been deprecated in OpenVINO 1.0 API." + " The last version which supports this is 2023.0")); +#elif INF_ENGINE_RELEASE >= 2021010000 return IE::make_shared_blob(y_blob, uv_blob); #else return IE::make_shared_blob(y_blob, uv_blob); @@ -982,7 +986,14 @@ static void setBlob(InferenceEngine::InferRequest& req, req.SetBlob(layer_name, blob); } else { GAPI_Assert(ctx.uu.params.kind == ParamDesc::Kind::Import); +#if INF_ENGINE_RELEASE > 2023000000 + // NB: SetBlob overload which accepts IE::PreProcessInfo + // has been deprecated - preprocessing can't be configured + // for "Import" networks anymore. + req.SetBlob(layer_name, blob); +#else req.SetBlob(layer_name, blob, ctx.uu.preproc_map.at(layer_name)); +#endif } } @@ -1370,7 +1381,14 @@ static void cfgImagePreprocessing(const IE::InputInfo::Ptr &ii, if (cv::util::holds_alternative(mm)) { const auto &meta = util::get(mm); if (meta.fmt == cv::MediaFormat::NV12) { +#if INF_ENGINE_RELEASE > 2023000000 + cv::util::throw_error(std::logic_error( + "IE Backend: cv::MediaFrame with NV12 format is no longer supported" + " because NV12 feature has been deprecated in OpenVINO 1.0 API." + " The last version which supports this is 2023.0")); +#else ii->getPreProcess().setColorFormat(IE::ColorFormat::NV12); +#endif } } } @@ -1426,7 +1444,14 @@ static IE::PreProcessInfo createImagePreProcInfo(const cv::GMetaArg &mm, if (cv::util::holds_alternative(mm)) { const auto &meta = util::get(mm); if (meta.fmt == cv::MediaFormat::NV12) { +#if INF_ENGINE_RELEASE > 2023000000 + cv::util::throw_error(std::logic_error( + "IE Backend: cv::MediaFrame with NV12 format is no longer supported" + " because NV12 feature has been deprecated in OpenVINO 1.0 API." + " The last version which supports this is 2023.0")); +#else info.setColorFormat(IE::ColorFormat::NV12); +#endif } } return info; @@ -2299,7 +2324,11 @@ IE::Blob::Ptr cv::gapi::ie::util::to_ie(const cv::Mat &blob) { IE::Blob::Ptr cv::gapi::ie::util::to_ie(const cv::Mat &y_plane, const cv::Mat &uv_plane) { auto y_blob = wrapIE(y_plane, cv::gapi::ie::TraitAs::IMAGE); auto uv_blob = wrapIE(uv_plane, cv::gapi::ie::TraitAs::IMAGE); -#if INF_ENGINE_RELEASE >= 2021010000 +#if INF_ENGINE_RELEASE > 2023000000 + cv::util::throw_error(std::logic_error( + "IE Backend: NV12 feature has been deprecated in OpenVINO 1.0 API." + " The last version which supports this is 2023.0")); +#elif INF_ENGINE_RELEASE >= 2021010000 return IE::make_shared_blob(y_blob, uv_blob); #else return IE::make_shared_blob(y_blob, uv_blob); diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp index 4056dd323f..58e37040e8 100644 --- a/modules/gapi/test/infer/gapi_infer_ie_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_ie_test.cpp @@ -62,6 +62,11 @@ public: return cv::MediaFrame::View(std::move(pp), std::move(ss), Cb{m_cb}); } cv::util::any blobParams() const override { +#if INF_ENGINE_RELEASE > 2023000000 + // NB: blobParams() shouldn't be used in tests + // if OpenVINO versions is higher than 2023.0 + GAPI_Assert(false && "NV12 feature has been deprecated in OpenVINO 1.0 API."); +#else return std::make_pair({IE::Precision::U8, {1, 3, 300, 300}, @@ -69,6 +74,7 @@ public: {{"HELLO", 42}, {"COLOR_FORMAT", InferenceEngine::ColorFormat::NV12}}); +#endif // INF_ENGINE_RELEASE > 2023000000 } }; @@ -138,7 +144,13 @@ void setNetParameters(IE::CNNNetwork& net, bool is_nv12 = false) { ii->setPrecision(IE::Precision::U8); ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); if (is_nv12) { +#if INF_ENGINE_RELEASE > 2023000000 + // NB: NV12 feature shouldn't be used in tests + // if OpenVINO versions is higher than 2023.0 + GAPI_Assert(false && "NV12 feature has been deprecated in OpenVINO 1.0 API."); +#else ii->getPreProcess().setColorFormat(IE::ColorFormat::NV12); +#endif // INF_ENGINE_RELEASE > 2023000000 } } @@ -392,10 +404,14 @@ struct InferWithReshapeNV12: public InferWithReshape { cv::randu(m_in_y, 0, 255); m_in_uv = cv::Mat{sz / 2, CV_8UC2}; cv::randu(m_in_uv, 0, 255); +// NB: NV12 feature shouldn't be used in tests +// if OpenVINO versions is higher than 2023.0 +#if INF_ENGINE_RELEASE <= 2023000000 setNetParameters(net, true); net.reshape({{"data", reshape_dims}}); auto frame_blob = cv::gapi::ie::util::to_ie(m_in_y, m_in_uv); inferROIs(frame_blob); +#endif // INF_ENGINE_RELEASE <= 2023000000 } }; @@ -505,8 +521,11 @@ struct ROIListNV12: public ::testing::Test { cv::Rect(cv::Point{50, 32}, cv::Size{128, 160}), }; - // Load & run IE network +// NB: NV12 feature shouldn't be used in tests +// if OpenVINO versions is higher than 2023.0 +#if INF_ENGINE_RELEASE <= 2023000000 { + // Load & run IE network auto plugin = cv::gimpl::ie::wrap::getPlugin(params); auto net = cv::gimpl::ie::wrap::readNetwork(params); setNetParameters(net, true); @@ -530,9 +549,11 @@ struct ROIListNV12: public ::testing::Test { m_out_ie_genders.push_back(to_ocv(infer_request.GetBlob("prob")).clone()); } } // namespace IE = .. +#endif // INF_ENGINE_RELEASE <= 2023000000 } // ROIList() void validate() { +#if INF_ENGINE_RELEASE <= 2023000000 // Validate with IE itself (avoid DNN module dependency here) ASSERT_EQ(2u, m_out_ie_ages.size()); ASSERT_EQ(2u, m_out_ie_genders.size()); @@ -543,6 +564,10 @@ struct ROIListNV12: public ::testing::Test { normAssert(m_out_ie_genders[0], m_out_gapi_genders[0], "0: Test gender output"); normAssert(m_out_ie_ages [1], m_out_gapi_ages [1], "1: Test age output"); normAssert(m_out_ie_genders[1], m_out_gapi_genders[1], "1: Test gender output"); +#else + GAPI_Assert(false && "Reference hasn't been calculated because" + " NV12 feature has been deprecated."); +#endif // INF_ENGINE_RELEASE <= 2023000000 } }; @@ -631,6 +656,9 @@ struct SingleROINV12: public ::testing::Test { m_roi = cv::Rect(cv::Point{64, 60}, cv::Size{96, 96}); +// NB: NV12 feature shouldn't be used in tests +// if OpenVINO versions is higher than 2023.0 +#if INF_ENGINE_RELEASE <= 2023000000 // Load & run IE network IE::Blob::Ptr ie_age, ie_gender; { @@ -657,12 +685,18 @@ struct SingleROINV12: public ::testing::Test { m_out_ie_age = to_ocv(infer_request.GetBlob("age_conv3")).clone(); m_out_ie_gender = to_ocv(infer_request.GetBlob("prob")).clone(); } +#endif // INF_ENGINE_RELEASE <= 2023000000 } void validate() { +#if INF_ENGINE_RELEASE <= 2023000000 // Validate with IE itself (avoid DNN module dependency here) normAssert(m_out_ie_age , m_out_gapi_age , "Test age output"); normAssert(m_out_ie_gender, m_out_gapi_gender, "Test gender output"); +#else + GAPI_Assert(false && "Reference hasn't been calculated because" + " NV12 feature has been deprecated."); +#endif } }; @@ -962,11 +996,20 @@ TEST_F(ROIListNV12, MediaInputNV12) auto pp = cv::gapi::ie::Params { params.model_path, params.weights_path, params.device_id }.cfgOutputLayers({ "age_conv3", "prob" }); + +// NB: NV12 feature has been deprecated in OpenVINO versions higher +// than 2023.0 so G-API must throw error in that case. +#if INF_ENGINE_RELEASE <= 2023000000 comp.apply(cv::gin(frame, m_roi_list), cv::gout(m_out_gapi_ages, m_out_gapi_genders), cv::compile_args(cv::gapi::networks(pp))); validate(); +#else + EXPECT_ANY_THROW(comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp)))); +#endif } TEST(TestAgeGenderIE, MediaInputNV12) @@ -986,6 +1029,9 @@ TEST(TestAgeGenderIE, MediaInputNV12) cv::Mat gapi_age, gapi_gender; +// NB: NV12 feature shouldn't be used in tests +// if OpenVINO versions is higher than 2023.0 +#if INF_ENGINE_RELEASE <= 2023000000 // Load & run IE network IE::Blob::Ptr ie_age, ie_gender; { @@ -999,6 +1045,7 @@ TEST(TestAgeGenderIE, MediaInputNV12) ie_age = infer_request.GetBlob("age_conv3"); ie_gender = infer_request.GetBlob("prob"); } +#endif // Configure & run G-API using AGInfo = std::tuple; @@ -1014,13 +1061,20 @@ TEST(TestAgeGenderIE, MediaInputNV12) auto pp = cv::gapi::ie::Params { params.model_path, params.weights_path, params.device_id }.cfgOutputLayers({ "age_conv3", "prob" }); + +// NB: NV12 feature has been deprecated in OpenVINO versions higher +// than 2023.0 so G-API must throw error in that case. +#if INF_ENGINE_RELEASE <= 2023000000 comp.apply(cv::gin(frame), cv::gout(gapi_age, gapi_gender), cv::compile_args(cv::gapi::networks(pp))); - // Validate with IE itself (avoid DNN module dependency here) normAssert(cv::gapi::ie::util::to_ocv(ie_age), gapi_age, "Test age output" ); normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output"); +#else + EXPECT_ANY_THROW(comp.apply(cv::gin(frame), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp)))); +#endif } TEST(TestAgeGenderIE, MediaInputBGR) @@ -1155,6 +1209,9 @@ TEST(InferROI, MediaInputNV12) cv::Mat gapi_age, gapi_gender; cv::Rect rect(cv::Point{64, 60}, cv::Size{96, 96}); +// NB: NV12 feature shouldn't be used in tests +// if OpenVINO versions is higher than 2023.0 +#if INF_ENGINE_RELEASE <= 2023000000 // Load & run IE network IE::Blob::Ptr ie_age, ie_gender; { @@ -1176,6 +1233,7 @@ TEST(InferROI, MediaInputNV12) ie_age = infer_request.GetBlob("age_conv3"); ie_gender = infer_request.GetBlob("prob"); } +#endif // Configure & run G-API using AGInfo = std::tuple; @@ -1192,13 +1250,20 @@ TEST(InferROI, MediaInputNV12) auto pp = cv::gapi::ie::Params { params.model_path, params.weights_path, params.device_id }.cfgOutputLayers({ "age_conv3", "prob" }); + +// NB: NV12 feature has been deprecated in OpenVINO versions higher +// than 2023.0 so G-API must throw error in that case. +#if INF_ENGINE_RELEASE <= 2023000000 comp.apply(cv::gin(frame, rect), cv::gout(gapi_age, gapi_gender), cv::compile_args(cv::gapi::networks(pp))); - // Validate with IE itself (avoid DNN module dependency here) normAssert(cv::gapi::ie::util::to_ocv(ie_age), gapi_age, "Test age output" ); normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output"); +#else + EXPECT_ANY_THROW(comp.apply(cv::gin(frame, rect), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp)))); +#endif } TEST_F(ROIList, Infer2MediaInputBGR) @@ -1233,10 +1298,20 @@ TEST_F(ROIListNV12, Infer2MediaInputNV12) auto pp = cv::gapi::ie::Params { params.model_path, params.weights_path, params.device_id }.cfgOutputLayers({ "age_conv3", "prob" }); + +// NB: NV12 feature has been deprecated in OpenVINO versions higher +// than 2023.0 so G-API must throw error in that case. +#if INF_ENGINE_RELEASE <= 2023000000 comp.apply(cv::gin(frame, m_roi_list), cv::gout(m_out_gapi_ages, m_out_gapi_genders), cv::compile_args(cv::gapi::networks(pp))); + validate(); +#else + EXPECT_ANY_THROW(comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp)))); +#endif } TEST_F(SingleROI, GenericInfer) @@ -1310,10 +1385,19 @@ TEST_F(SingleROINV12, GenericInferMediaNV12) pp.cfgNumRequests(2u); auto frame = MediaFrame::Create(m_in_y, m_in_uv); + +// NB: NV12 feature has been deprecated in OpenVINO versions higher +// than 2023.0 so G-API must throw error in that case. +#if INF_ENGINE_RELEASE <= 2023000000 comp.apply(cv::gin(frame, m_roi), cv::gout(m_out_gapi_age, m_out_gapi_gender), cv::compile_args(cv::gapi::networks(pp))); validate(); +#else + EXPECT_ANY_THROW(comp.apply(cv::gin(frame, m_roi), + cv::gout(m_out_gapi_age, m_out_gapi_gender), + cv::compile_args(cv::gapi::networks(pp)))); +#endif } TEST_F(ROIList, GenericInfer) @@ -1386,11 +1470,20 @@ TEST_F(ROIListNV12, GenericInferMediaNV12) pp.cfgNumRequests(2u); auto frame = MediaFrame::Create(m_in_y, m_in_uv); + + // NB: NV12 feature has been deprecated in OpenVINO versions higher + // than 2023.0 so G-API must throw error in that case. +#if INF_ENGINE_RELEASE <= 2023000000 comp.apply(cv::gin(frame, m_roi_list), - cv::gout(m_out_gapi_ages, m_out_gapi_genders), - cv::compile_args(cv::gapi::networks(pp))); + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp))); validate(); +#else + EXPECT_ANY_THROW(comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp)))); +#endif } TEST_F(ROIList, GenericInfer2) @@ -1461,10 +1554,20 @@ TEST_F(ROIListNV12, GenericInfer2MediaInputNV12) pp.cfgNumRequests(2u); auto frame = MediaFrame::Create(m_in_y, m_in_uv); + +// NB: NV12 feature has been deprecated in OpenVINO versions higher +// than 2023.0 so G-API must throw error in that case. +#if INF_ENGINE_RELEASE <= 2023000000 comp.apply(cv::gin(frame, m_roi_list), - cv::gout(m_out_gapi_ages, m_out_gapi_genders), - cv::compile_args(cv::gapi::networks(pp))); + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp))); + validate(); +#else + EXPECT_ANY_THROW(comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp)))); +#endif } TEST(Infer, SetInvalidNumberOfRequests) @@ -2050,11 +2153,20 @@ TEST_F(InferWithReshapeNV12, TestInferListYUV) auto pp = cv::gapi::ie::Params { params.model_path, params.weights_path, params.device_id }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}}); + +// NB: NV12 feature has been deprecated in OpenVINO versions higher +// than 2023.0 so G-API must throw error in that case. +#if INF_ENGINE_RELEASE <= 2023000000 comp.apply(cv::gin(frame, m_roi_list), cv::gout(m_out_gapi_ages, m_out_gapi_genders), cv::compile_args(cv::gapi::networks(pp))); // Validate validate(); +#else + EXPECT_ANY_THROW(comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp)))); +#endif } TEST_F(ROIList, CallInferMultipleTimes) @@ -2079,6 +2191,7 @@ TEST_F(ROIList, CallInferMultipleTimes) validate(); } +#if INF_ENGINE_RELEASE <= 2023000000 TEST(IEFrameAdapter, blobParams) { cv::Mat bgr = cv::Mat::eye(240, 320, CV_8UC3); @@ -2093,6 +2206,7 @@ TEST(IEFrameAdapter, blobParams) EXPECT_EQ(expected, actual); } +#endif namespace { @@ -2281,6 +2395,10 @@ TEST(TestAgeGenderIE, InferWithBatch) normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output"); } +// NB: All tests below use preprocessing for "Import" networks +// passed as the last argument to SetBLob. This overload has +// been deprecated in OpenVINO 1.0 API. +#if INF_ENGINE_RELEASE <= 2023000000 TEST(ImportNetwork, Infer) { const std::string device = "MYRIAD"; @@ -2820,6 +2938,7 @@ TEST(ImportNetwork, InferList2NV12) normAssert(out_ie_genders[i], out_gapi_genders[i], "Test gender output"); } } +#endif TEST(TestAgeGender, ThrowBlobAndInputPrecisionMismatch) { From 71790e12adb202044bcab2b181442d4f5261c26c Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Tue, 20 Jun 2023 11:29:23 +0100 Subject: [PATCH 022/105] Merge pull request #23799 from TolyaTalamanov:at/ov20-backend-implement-missing-kernels G-API: Implement InferROI, InferList, InferList2 for OpenVINO backend #23799 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/gapi/src/backends/ov/govbackend.cpp | 759 ++++++++++++++---- modules/gapi/src/backends/ov/util.hpp | 2 + .../gapi/test/infer/gapi_infer_ov_tests.cpp | 459 +++++++---- 3 files changed, 904 insertions(+), 316 deletions(-) diff --git a/modules/gapi/src/backends/ov/govbackend.cpp b/modules/gapi/src/backends/ov/govbackend.cpp index 46eccd2bbd..0fff90bb4c 100644 --- a/modules/gapi/src/backends/ov/govbackend.cpp +++ b/modules/gapi/src/backends/ov/govbackend.cpp @@ -101,8 +101,8 @@ static int toCV(const ov::element::Type &type) { static void copyFromOV(const ov::Tensor &tensor, cv::Mat &mat) { const auto total = mat.total() * mat.channels(); - if (tensor.get_element_type() != toOV(mat.depth()) || - tensor.get_size() != total ) { + if (toCV(tensor.get_element_type()) != mat.depth() || + tensor.get_size() != total ) { std::stringstream ss; ss << "Failed to copy data from ov::Tensor to cv::Mat." << " Data type or number of elements mismatch." @@ -128,8 +128,8 @@ static void copyToOV(const cv::Mat &mat, ov::Tensor &tensor) { // TODO: Ideally there should be check that mat and tensor // dimensions are compatible. const auto total = mat.total() * mat.channels(); - if (tensor.get_element_type() != toOV(mat.depth()) || - tensor.get_size() != total) { + if (toCV(tensor.get_element_type()) != mat.depth() || + tensor.get_size() != total) { std::stringstream ss; ss << "Failed to copy data from cv::Mat to ov::Tensor." << " Data type or number of elements mismatch." @@ -158,6 +158,14 @@ int cv::gapi::ov::util::to_ocv(const ::ov::element::Type &type) { return toCV(type); } +void cv::gapi::ov::util::to_ov(const cv::Mat &mat, ::ov::Tensor &tensor) { + copyToOV(mat, tensor); +} + +void cv::gapi::ov::util::to_ocv(const ::ov::Tensor &tensor, cv::Mat &mat) { + copyFromOV(tensor, mat); +} + struct OVUnit { static const char *name() { return "OVUnit"; } @@ -343,6 +351,15 @@ cv::GArg OVCallContext::packArg(const cv::GArg &arg) { switch (ref.shape) { case cv::GShape::GMAT: return cv::GArg(m_res.slot()[ref.id]); + + // Note: .at() is intentional for GArray as object MUST be already there + // (and constructed by either bindIn/Out or resetInternal) + case cv::GShape::GARRAY: return cv::GArg(m_res.slot().at(ref.id)); + + // Note: .at() is intentional for GOpaque as object MUST be already there + // (and constructed by either bindIn/Out or resetInternal) + case cv::GShape::GOPAQUE: return cv::GArg(m_res.slot().at(ref.id)); + default: cv::util::throw_error(std::logic_error("Unsupported GShape type")); break; @@ -547,6 +564,62 @@ static void PostOutputs(::ov::InferRequest &infer_request, } } +class PostOutputsList { +public: + PostOutputsList(size_t size, + std::shared_ptr ctx); + + void operator()(::ov::InferRequest &infer_request, + std::exception_ptr eptr, + size_t pos) const; + +private: + struct Priv { + std::atomic finished{0u}; + size_t size; + std::shared_ptr ctx; + }; + std::shared_ptr m_priv; +}; + +PostOutputsList::PostOutputsList(size_t size, + std::shared_ptr ctx) + : m_priv(new Priv{}) { + m_priv->size = size; + m_priv->ctx = ctx; +} + +void PostOutputsList::operator()(::ov::InferRequest &infer_request, + std::exception_ptr eptr, + size_t pos) const { + auto&& ctx = m_priv->ctx; + auto&& finished = m_priv->finished; + auto&& size = m_priv->size; + + ctx->eptr = eptr; + if (!ctx->eptr) { + for (auto i : ade::util::iota(ctx->uu.params.num_out)) { + std::vector &out_vec = ctx->outVecR(i); + + const auto &out_name = ctx->uu.params.output_names[i]; + const auto &out_tensor = infer_request.get_tensor(out_name); + + out_vec[pos].create(toCV(out_tensor.get_shape()), + toCV(out_tensor.get_element_type())); + copyFromOV(out_tensor, out_vec[pos]); + } + } + ++finished; + + if (finished == size) { + for (auto i : ade::util::iota(ctx->uu.params.num_out)) { + auto output = ctx->output(i); + ctx->out.meta(output, ctx->getMeta()); + ctx->out.post(std::move(output), ctx->eptr); + } + } +} + namespace cv { namespace gimpl { namespace ov { @@ -594,6 +667,34 @@ cv::optional lookUp(const std::map &map, const K& key) { return cv::util::make_optional(std::move(it->second)); } +// NB: This function is used to preprocess input image +// for InferROI, InferList, InferList2 kernels. +static cv::Mat preprocess(const cv::Mat &in_mat, + const cv::Rect &roi, + const ::ov::Shape &model_shape) { + cv::Mat out; + // FIXME: Since there is no information about H and W positions + // among tensor dimmensions assume that model layout is "NHWC". + // (In fact "NHWC" is the only right layout for preprocessing because + // it works only with images. + GAPI_Assert(model_shape.size() == 4u); + const auto H = model_shape[1]; + const auto W = model_shape[2]; + const auto C = model_shape[3]; + // NB: Soft check that at least number of channels matches. + if (static_cast(C) != in_mat.channels()) { + std::stringstream ss; + ss << "OV Backend: Failed to preprocess input data " + " (Number of channels mismatch)." + " Provided data: " << cv::descr_of(in_mat) << + " and Model shape: " << model_shape; + util::throw_error(std::logic_error(ss.str())); + } + // NB: Crop roi and resize to model size. + cv::resize(in_mat(roi), out, cv::Size(W, H)); + return out; +} + static bool isImage(const cv::GMatDesc &desc, const ::ov::Shape &model_shape) { return (model_shape.size() == 4u) && @@ -603,6 +704,191 @@ static bool isImage(const cv::GMatDesc &desc, (desc.depth == CV_8U); } +class PrePostProcWrapper { +public: + PrePostProcWrapper(std::shared_ptr<::ov::Model> &model, + const ParamDesc::Model &model_info, + const std::vector &input_names, + const std::vector &output_names) + : m_ppp(model), + m_model(model), + m_model_info(model_info), + m_input_names(input_names), + m_output_names(output_names) { + // NB: Do Reshape right away since it must be the first step of model modification + // and applicable for all infer kernels. + const auto new_shapes = broadcastLayerAttr(model_info.new_shapes, input_names); + m_model->reshape(toOV(new_shapes)); + + const auto &mi = m_model_info; + m_input_tensor_layout = broadcastLayerAttr(mi.input_tensor_layout, m_input_names); + m_input_model_layout = broadcastLayerAttr(mi.input_model_layout, m_input_names); + m_interpolation = broadcastLayerAttr(mi.interpolation, m_input_names); + m_mean_values = broadcastLayerAttr(mi.mean_values, m_input_names); + m_scale_values = broadcastLayerAttr(mi.scale_values, m_input_names); + m_interpolation = broadcastLayerAttr(mi.interpolation, m_input_names); + + m_output_tensor_layout = broadcastLayerAttr(mi.output_tensor_layout, m_output_names); + m_output_model_layout = broadcastLayerAttr(mi.output_model_layout, m_output_names); + m_output_tensor_precision = broadcastLayerAttr(mi.output_tensor_precision, m_output_names); + }; + + void cfgLayouts(const std::string &input_name) { + auto &input_info = m_ppp.input(input_name); + const auto explicit_in_model_layout = lookUp(m_input_model_layout, input_name); + if (explicit_in_model_layout) { + input_info.model().set_layout(::ov::Layout(*explicit_in_model_layout)); + } + const auto explicit_in_tensor_layout = lookUp(m_input_tensor_layout, input_name); + if (explicit_in_tensor_layout) { + input_info.tensor().set_layout(::ov::Layout(*explicit_in_tensor_layout)); + } + } + + void cfgScaleMean(const std::string &input_name) { + auto &input_info = m_ppp.input(input_name); + const auto mean_vec = lookUp(m_mean_values, input_name); + if (mean_vec) { + input_info.preprocess().mean(*mean_vec); + } + const auto scale_vec = lookUp(m_scale_values, input_name); + if (scale_vec) { + input_info.preprocess().scale(*scale_vec); + } + } + + // FIXME: Decompose this... + void cfgPreProcessing(const std::string &input_name, + const cv::GMetaArg &input_meta, + const bool disable_img_resize = false) { + GAPI_Assert(cv::util::holds_alternative(input_meta)); + const auto &matdesc = cv::util::get(input_meta); + + const auto explicit_in_tensor_layout = lookUp(m_input_tensor_layout, input_name); + const auto explicit_resize = lookUp(m_interpolation, input_name); + + if (disable_img_resize && explicit_resize.has_value()) { + std::stringstream ss; + util::throw_error(std::logic_error( + "OV Backend: Resize for layer \"" + input_name + "\" will be performed" + " on host via OpenCV so explicitly configured resize is prohibited.")); + } + + const auto &input_shape = m_model->input(input_name).get_shape(); + auto &input_info = m_ppp.input(input_name); + + m_ppp.input(input_name).tensor().set_element_type(toOV(matdesc.depth)); + if (isImage(matdesc, input_shape)) { + // NB: Image case - all necessary preprocessng is configured automatically. + GAPI_LOG_DEBUG(NULL, "OV Backend: Input: \"" << input_name << "\" is image."); + if (explicit_in_tensor_layout && + *explicit_in_tensor_layout != "NHWC") { + std::stringstream ss; + ss << "OV Backend: Provided tensor layout " << *explicit_in_tensor_layout + << " is not compatible with input data " << matdesc << " for layer \"" + << input_name << "\". Expecting NHWC"; + util::throw_error(std::logic_error(ss.str())); + } else { + input_info.tensor().set_layout(::ov::Layout("NHWC")); + } + + if (!disable_img_resize) { + input_info.tensor().set_spatial_static_shape(matdesc.size.height, + matdesc.size.width); + // NB: Even though resize is automatically configured + // user have an opportunity to specify the interpolation algorithm. + auto interp = explicit_resize + ? toOVInterp(*explicit_resize) + : ::ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR; + input_info.preprocess().resize(interp); + } + } else { + // NB: Tensor case - resize or layout conversions must be explicitly specified. + GAPI_LOG_DEBUG(NULL, "OV Backend: Input: \"" << input_name << "\" is tensor."); + + if (explicit_resize) { + if (matdesc.isND()) { + // NB: ND case - need to obtain "H" and "W" positions + // in order to configure resize. + const auto model_layout = ::ov::layout::get_layout(m_model->input(input_name)); + if (!explicit_in_tensor_layout && model_layout.empty()) { + std::stringstream ss; + ss << "Resize for input layer: " << input_name + << "can't be configured." + << " Failed to extract H and W positions from layout."; + util::throw_error(std::logic_error(ss.str())); + } else { + const auto layout = explicit_in_tensor_layout + ? ::ov::Layout(*explicit_in_tensor_layout) : model_layout; + auto H_idx = ::ov::layout::height_idx(layout); + auto W_idx = ::ov::layout::width_idx(layout); + // NB: If layout is "...HW", H position is -2. + if (H_idx < 0) H_idx = matdesc.dims.size() + H_idx; + if (W_idx < 0) W_idx = matdesc.dims.size() + W_idx; + GAPI_Assert(H_idx >= 0 && H_idx < static_cast(matdesc.dims.size())); + GAPI_Assert(W_idx >= 0 && W_idx < static_cast(matdesc.dims.size())); + input_info.tensor().set_spatial_static_shape(matdesc.dims[H_idx], + matdesc.dims[W_idx]); + input_info.preprocess().resize(toOVInterp(*explicit_resize)); + } + } else { + // NB: 2D case - We know exactly where H and W... + input_info.tensor().set_spatial_static_shape(matdesc.size.height, + matdesc.size.width); + input_info.preprocess().resize(toOVInterp(*explicit_resize)); + } + } + } + } + + void cfgPostProcessing() { + for (const auto &output_name : m_output_names) { + const auto explicit_out_tensor_layout = + lookUp(m_output_tensor_layout, output_name); + if (explicit_out_tensor_layout) { + m_ppp.output(output_name).tensor() + .set_layout(::ov::Layout(*explicit_out_tensor_layout)); + } + + const auto explicit_out_model_layout = + lookUp(m_output_model_layout, output_name); + if (explicit_out_model_layout) { + m_ppp.output(output_name).model() + .set_layout(::ov::Layout(*explicit_out_model_layout)); + } + + const auto explicit_out_tensor_prec = + lookUp(m_output_tensor_precision, output_name); + if (explicit_out_tensor_prec) { + m_ppp.output(output_name).tensor() + .set_element_type(toOV(*explicit_out_tensor_prec)); + } + } + } + + void finalize() { + GAPI_LOG_DEBUG(NULL, "OV Backend: PrePostProcessor: " << m_ppp); + m_model = m_ppp.build(); + } + +private: + ::ov::preprocess::PrePostProcessor m_ppp; + + std::shared_ptr<::ov::Model> &m_model; + const ParamDesc::Model &m_model_info; + const std::vector &m_input_names; + const std::vector &m_output_names; + + cv::gimpl::ov::AttrMap m_input_tensor_layout; + cv::gimpl::ov::AttrMap m_input_model_layout; + cv::gimpl::ov::AttrMap m_interpolation; + cv::gimpl::ov::AttrMap> m_mean_values; + cv::gimpl::ov::AttrMap> m_scale_values; + cv::gimpl::ov::AttrMap m_output_tensor_layout; + cv::gimpl::ov::AttrMap m_output_model_layout; + cv::gimpl::ov::AttrMap m_output_tensor_precision; +}; + struct Infer: public cv::detail::KernelTag { using API = cv::GInferBase; static cv::gapi::GBackend backend() { return cv::gapi::ov::backend(); } @@ -625,156 +911,21 @@ struct Infer: public cv::detail::KernelTag { // NB: Pre/Post processing configuration avaiable only for read models. if (cv::util::holds_alternative(uu.params.kind)) { const auto &model_info = cv::util::get(uu.params.kind); - const auto new_shapes = - broadcastLayerAttr(model_info.new_shapes, - uu.params.input_names); - const_cast&>(uu.model)->reshape(toOV(new_shapes)); + auto& model = const_cast&>(uu.model); + PrePostProcWrapper ppp {model, model_info, + uu.params.input_names, uu.params.output_names}; - const auto input_tensor_layout = - broadcastLayerAttr(model_info.input_tensor_layout, - uu.params.input_names); - const auto input_model_layout = - broadcastLayerAttr(model_info.input_model_layout, - uu.params.input_names); - - const auto interpolation = broadcastLayerAttr(model_info.interpolation, - uu.params.input_names); - const auto mean_values = broadcastLayerAttr(model_info.mean_values, - uu.params.input_names); - const auto scale_values = broadcastLayerAttr(model_info.scale_values, - uu.params.input_names); - // FIXME: Pre/Post processing step shouldn't be configured in this method. - ::ov::preprocess::PrePostProcessor ppp(uu.model); for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names), ade::util::toRange(in_metas))) { - const auto &mm = std::get<1>(it); - GAPI_Assert(cv::util::holds_alternative(mm)); - const auto &matdesc = cv::util::get(mm); - const auto &input_name = std::get<0>(it); - auto &input_info = ppp.input(input_name); - input_info.tensor().set_element_type(toOV(matdesc.depth)); + const auto &mm = std::get<1>(it); - const auto explicit_in_model_layout = lookUp(input_model_layout, input_name); - if (explicit_in_model_layout) { - input_info.model().set_layout(::ov::Layout(*explicit_in_model_layout)); - } - const auto explicit_in_tensor_layout = lookUp(input_tensor_layout, input_name); - if (explicit_in_tensor_layout) { - input_info.tensor().set_layout(::ov::Layout(*explicit_in_tensor_layout)); - } - const auto explicit_resize = lookUp(interpolation, input_name); - // NB: Note that model layout still can't be empty. - // e.g If model converted to IRv11 without any additional - // info about layout via Model Optimizer. - const auto model_layout = ::ov::layout::get_layout(uu.model->input(input_name)); - const auto &input_shape = uu.model->input(input_name).get_shape(); - if (isImage(matdesc, input_shape)) { - // NB: Image case - all necessary preprocessng is configured automatically. - GAPI_LOG_DEBUG(NULL, "OV Backend: Input: \"" << input_name << "\" is image."); - // NB: Layout is already set just double check that - // user provided the correct one. In fact, there is only one correct for image. - if (explicit_in_tensor_layout && - *explicit_in_tensor_layout != "NHWC") { - std::stringstream ss; - ss << "OV Backend: Provided tensor layout " << *explicit_in_tensor_layout - << " is not compatible with input data " << matdesc << " for layer \"" - << input_name << "\". Expecting NHWC"; - util::throw_error(std::logic_error(ss.str())); - } - input_info.tensor().set_layout(::ov::Layout("NHWC")); - input_info.tensor().set_spatial_static_shape(matdesc.size.height, - matdesc.size.width); - // NB: Even though resize is automatically configured - // user have an opportunity to specify the interpolation algorithm. - auto interp = explicit_resize - ? toOVInterp(*explicit_resize) - : ::ov::preprocess::ResizeAlgorithm::RESIZE_LINEAR; - input_info.preprocess().resize(interp); - } else { - // NB: Tensor case - resize or layout conversions must be explicitly specified. - GAPI_LOG_DEBUG(NULL, "OV Backend: Input: \"" << input_name << "\" is tensor."); - if (explicit_resize) { - if (matdesc.isND()) { - // NB: ND case - need to obtain "H" and "W" positions - // in order to configure resize. - if (!explicit_in_tensor_layout && model_layout.empty()) { - std::stringstream ss; - ss << "Resize for input layer: " << input_name - << "can't be configured." - << " Failed to extract H and W positions from layout."; - util::throw_error(std::logic_error(ss.str())); - } else { - const auto layout = explicit_in_tensor_layout - ? ::ov::Layout(*explicit_in_tensor_layout) : model_layout; - auto H_idx = ::ov::layout::height_idx(layout); - auto W_idx = ::ov::layout::width_idx(layout); - // NB: If layout is "...HW", H position is -2. - if (H_idx < 0) H_idx = matdesc.dims.size() + H_idx; - if (W_idx < 0) W_idx = matdesc.dims.size() + W_idx; - GAPI_Assert(H_idx >= 0 && H_idx < static_cast(matdesc.dims.size())); - GAPI_Assert(W_idx >= 0 && W_idx < static_cast(matdesc.dims.size())); - input_info.tensor().set_spatial_static_shape(matdesc.dims[H_idx], - matdesc.dims[W_idx]); - input_info.preprocess().resize(toOVInterp(*explicit_resize)); - } - } else { - // NB: 2D case - We know exactly where H and W... - input_info.tensor().set_spatial_static_shape(matdesc.size.height, - matdesc.size.width); - input_info.preprocess().resize(toOVInterp(*explicit_resize)); - } - } - } - // NB: Apply mean/scale as the last step of the preprocessing. - // Note that this can be applied to any input data if the - // position of "C" dimension is known. - const auto mean_vec = lookUp(mean_values, input_name); - if (mean_vec) { - input_info.preprocess().mean(*mean_vec); - } - - const auto scale_vec = lookUp(scale_values, input_name); - if (scale_vec) { - input_info.preprocess().scale(*scale_vec); - } + ppp.cfgLayouts(input_name); + ppp.cfgPreProcessing(input_name, mm); + ppp.cfgScaleMean(input_name); } - - const auto output_tensor_layout = - broadcastLayerAttr(model_info.output_tensor_layout, - uu.params.output_names); - const auto output_model_layout = - broadcastLayerAttr(model_info.output_model_layout, - uu.params.output_names); - const auto output_tensor_precision = - broadcastLayerAttr(model_info.output_tensor_precision, - uu.params.output_names); - - for (const auto &output_name : uu.params.output_names) { - const auto explicit_out_tensor_layout = - lookUp(output_tensor_layout, output_name); - if (explicit_out_tensor_layout) { - ppp.output(output_name).tensor() - .set_layout(::ov::Layout(*explicit_out_tensor_layout)); - } - - const auto explicit_out_model_layout = - lookUp(output_model_layout, output_name); - if (explicit_out_model_layout) { - ppp.output(output_name).model() - .set_layout(::ov::Layout(*explicit_out_model_layout)); - } - - const auto explicit_out_tensor_prec = - lookUp(output_tensor_precision, output_name); - if (explicit_out_tensor_prec) { - ppp.output(output_name).tensor() - .set_element_type(toOV(*explicit_out_tensor_prec)); - } - } - - GAPI_LOG_DEBUG(NULL, "OV Backend: PrePostProcessor: " << ppp); - const_cast&>(uu.model) = ppp.build(); + ppp.cfgPostProcessing(); + ppp.finalize(); } for (const auto &out_name : uu.params.output_names) { @@ -815,6 +966,313 @@ struct Infer: public cv::detail::KernelTag { } }; +struct InferROI: public cv::detail::KernelTag { + using API = cv::GInferROIBase; + static cv::gapi::GBackend backend() { return cv::gapi::ov::backend(); } + static KImpl kernel() { return KImpl{outMeta, run}; } + + static cv::GMetaArgs outMeta(const ade::Graph &gr, + const ade::NodeHandle &nh, + const cv::GMetaArgs &in_metas, + const cv::GArgs &/*in_args*/) { + cv::GMetaArgs result; + + GConstGOVModel gm(gr); + const auto &uu = gm.metadata(nh).get(); + // Initialize input information + // FIXME: So far it is pretty limited + GAPI_Assert(1u == uu.params.input_names.size()); + GAPI_Assert(2u == in_metas.size()); + + const auto &input_name = uu.params.input_names.at(0); + const auto &mm = in_metas.at(1u); + GAPI_Assert(cv::util::holds_alternative(mm)); + const auto &matdesc = cv::util::get(mm); + + const bool is_model = cv::util::holds_alternative(uu.params.kind); + const auto &input_shape = is_model ? uu.model->input(input_name).get_shape() + : uu.compiled_model.input(input_name).get_shape(); + if (!isImage(matdesc, input_shape)) { + util::throw_error(std::runtime_error( + "OV Backend: InferROI supports only image as the 1th argument")); + } + + if (is_model) { + const auto &model_info = cv::util::get(uu.params.kind); + auto& model = const_cast&>(uu.model); + PrePostProcWrapper ppp {model, model_info, + uu.params.input_names, uu.params.output_names}; + + ppp.cfgLayouts(input_name); + ppp.cfgPreProcessing(input_name, mm, true /*disable_img_resize*/); + ppp.cfgScaleMean(input_name); + ppp.cfgPostProcessing(); + ppp.finalize(); + } + + for (const auto &out_name : uu.params.output_names) { + cv::GMatDesc outm; + if (cv::util::holds_alternative(uu.params.kind)) { + const auto &out = uu.model->output(out_name); + outm = cv::GMatDesc(toCV(out.get_element_type()), + toCV(out.get_shape())); + } else { + GAPI_Assert(cv::util::holds_alternative(uu.params.kind)); + const auto &out = uu.compiled_model.output(out_name); + outm = cv::GMatDesc(toCV(out.get_element_type()), + toCV(out.get_shape())); + } + result.emplace_back(std::move(outm)); + } + + return result; + } + + static void run(std::shared_ptr ctx, + cv::gimpl::ov::RequestPool &reqPool) { + using namespace std::placeholders; + reqPool.getIdleRequest()->execute( + IInferExecutor::Task { + [ctx](::ov::InferRequest &infer_request) { + GAPI_Assert(ctx->uu.params.num_in == 1); + const auto &input_name = ctx->uu.params.input_names[0]; + auto input_tensor = infer_request.get_tensor(input_name); + const auto &shape = input_tensor.get_shape(); + const auto &roi = ctx->inArg(0).rref(); + const auto roi_mat = preprocess(ctx->inMat(1), roi, shape); + copyToOV(roi_mat, input_tensor); + }, + std::bind(PostOutputs, _1, _2, ctx) + } + ); + } +}; + +struct InferList: public cv::detail::KernelTag { + using API = cv::GInferListBase; + static cv::gapi::GBackend backend() { return cv::gapi::ov::backend(); } + static KImpl kernel() { return KImpl{outMeta, run}; } + + static cv::GMetaArgs outMeta(const ade::Graph &gr, + const ade::NodeHandle &nh, + const cv::GMetaArgs &in_metas, + const cv::GArgs &/*in_args*/) { + GConstGOVModel gm(gr); + const auto &uu = gm.metadata(nh).get(); + // Initialize input information + // Note our input layers list order matches the API order and so + // meta order. + GAPI_Assert(uu.params.input_names.size() == (in_metas.size() - 1u) + && "Known input layers count doesn't match input meta count"); + + // NB: Pre/Post processing configuration avaiable only for read models. + if (cv::util::holds_alternative(uu.params.kind)) { + const auto &model_info = cv::util::get(uu.params.kind); + auto& model = const_cast&>(uu.model); + PrePostProcWrapper ppp {model, model_info, + uu.params.input_names, uu.params.output_names}; + + size_t idx = 1u; + for (auto &&input_name : uu.params.input_names) { + const auto &mm = in_metas[idx++]; + GAPI_Assert(cv::util::holds_alternative(mm)); + const auto &matdesc = cv::util::get(mm); + const auto &input_shape = uu.model->input(input_name).get_shape(); + + if (!isImage(matdesc, input_shape)) { + util::throw_error(std::runtime_error( + "OV Backend: Only image is supported" + " as the " + std::to_string(idx) + "th argument for InferList")); + } + + ppp.cfgLayouts(input_name); + ppp.cfgPreProcessing(input_name, mm, true /*disable_img_resize*/); + ppp.cfgScaleMean(input_name); + } + ppp.cfgPostProcessing(); + ppp.finalize(); + } + + // roi-list version is much easier at the moment. + // All our outputs are vectors which don't have + // metadata at the moment - so just create a vector of + // "empty" array metadatas of the required size. + return cv::GMetaArgs(uu.params.output_names.size(), + cv::GMetaArg{cv::empty_array_desc()}); + } + + static void run(std::shared_ptr ctx, + cv::gimpl::ov::RequestPool &reqPool) { + const auto& in_roi_vec = ctx->inArg(0u).rref(); + // NB: In case there is no input data need to post output anyway + if (in_roi_vec.empty()) { + for (auto i : ade::util::iota(ctx->uu.params.num_out)) { + auto output = ctx->output(i); + ctx->out.meta(output, ctx->getMeta()); + ctx->out.post(std::move(output)); + } + return; + } + + for (auto i : ade::util::iota(ctx->uu.params.num_out)) { + // FIXME: Isn't this should be done automatically + // by some resetInternalData(), etc? (Probably at the GExecutor level) + auto& out_vec = ctx->outVecR(i); + out_vec.clear(); + out_vec.resize(in_roi_vec.size()); + } + + PostOutputsList callback(in_roi_vec.size(), ctx); + for (auto&& it : ade::util::indexed(in_roi_vec)) { + const auto pos = ade::util::index(it); + const auto &rc = ade::util::value(it); + reqPool.getIdleRequest()->execute( + IInferExecutor::Task { + [ctx, rc](::ov::InferRequest &infer_request) { + const auto &input_name = ctx->uu.params.input_names[0]; + auto input_tensor = infer_request.get_tensor(input_name); + const auto &shape = input_tensor.get_shape(); + const auto roi_mat = preprocess(ctx->inMat(1), rc, shape); + copyToOV(roi_mat, input_tensor); + }, + std::bind(callback, std::placeholders::_1, std::placeholders::_2, pos) + } + ); + } + } +}; + +struct InferList2: public cv::detail::KernelTag { + using API = cv::GInferList2Base; + static cv::gapi::GBackend backend() { return cv::gapi::ov::backend(); } + static KImpl kernel() { return KImpl{outMeta, run}; } + + static cv::GMetaArgs outMeta(const ade::Graph &gr, + const ade::NodeHandle &nh, + const cv::GMetaArgs &in_metas, + const cv::GArgs &/*in_args*/) { + GConstGOVModel gm(gr); + const auto &uu = gm.metadata(nh).get(); + // Initialize input information + // Note our input layers list order matches the API order and so + // meta order. + GAPI_Assert(uu.params.input_names.size() == (in_metas.size() - 1u) + && "Known input layers count doesn't match input meta count"); + + const auto &op = gm.metadata(nh).get(); + + // In contrast to InferList, the InferList2 has only one + // "full-frame" image argument, and all the rest are arrays of + // ether ROI or blobs. So here we set the 0th arg image format + // to all inputs which are ROI-based (skipping the + // "blob"-based ones) + // FIXME: this is filtering not done, actually! GArrayDesc has + // no hint for its underlying type! + + const auto &input_name_0 = uu.params.input_names.front(); + const auto &mm_0 = in_metas[0u]; + const auto &matdesc = cv::util::get(mm_0); + + const bool is_model = cv::util::holds_alternative(uu.params.kind); + const auto &input_shape = is_model ? uu.model->input(input_name_0).get_shape() + : uu.compiled_model.input(input_name_0).get_shape(); + if (!isImage(matdesc, input_shape)) { + util::throw_error(std::runtime_error( + "OV Backend: InferList2 supports only image as the 0th argument")); + } + + if (is_model) { + const auto &model_info = cv::util::get(uu.params.kind); + auto& model = const_cast&>(uu.model); + PrePostProcWrapper ppp {model, model_info, + uu.params.input_names, uu.params.output_names}; + + size_t idx = 1u; + for (auto &&input_name : uu.params.input_names) { + GAPI_Assert(util::holds_alternative(in_metas[idx]) + && "Non-array inputs are not supported"); + + ppp.cfgLayouts(input_name); + if (op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_RECT) { + ppp.cfgPreProcessing(input_name, mm_0, true /*disable_img_resize*/); + } else { + // This is a cv::GMat (equals to: cv::Mat) + // Just validate that it is really the type + // (other types are prohibited here) + GAPI_Assert(op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_MAT); + } + + ppp.cfgScaleMean(input_name); + idx++; // NB: Never forget to increment the counter + } + ppp.cfgPostProcessing(); + ppp.finalize(); + } + + // roi-list version is much easier at the moment. + // All our outputs are vectors which don't have + // metadata at the moment - so just create a vector of + // "empty" array metadatas of the required size. + return cv::GMetaArgs(uu.params.output_names.size(), + cv::GMetaArg{cv::empty_array_desc()}); + } + + static void run(std::shared_ptr ctx, + cv::gimpl::ov::RequestPool &reqPool) { + GAPI_Assert(ctx->inArgs().size() > 1u + && "This operation must have at least two arguments"); + // NB: This blob will be used to make roi from its, so + // it should be treated as image + const auto list_size = ctx->inArg(1u).size(); + if (list_size == 0u) { + for (auto i : ade::util::iota(ctx->uu.params.num_out)) { + auto output = ctx->output(i); + ctx->out.meta(output, ctx->getMeta()); + ctx->out.post(std::move(output)); + } + return; + } + + for (auto i : ade::util::iota(ctx->uu.params.num_out)) { + // FIXME: Isn't this should be done automatically + // by some resetInternalData(), etc? (Probably at the GExecutor level) + auto& out_vec = ctx->outVecR(i); + out_vec.clear(); + out_vec.resize(list_size); + } + + PostOutputsList callback(list_size, ctx); + for (const auto &list_idx : ade::util::iota(list_size)) { + reqPool.getIdleRequest()->execute( + IInferExecutor::Task { + [ctx, list_idx, list_size](::ov::InferRequest &infer_request) { + for (auto in_idx : ade::util::iota(ctx->uu.params.num_in)) { + const auto &this_vec = ctx->inArg(in_idx+1u); + GAPI_Assert(this_vec.size() == list_size); + const auto &input_name = ctx->uu.params.input_names[in_idx]; + auto input_tensor = infer_request.get_tensor(input_name); + const auto &shape = input_tensor.get_shape(); + if (this_vec.getKind() == cv::detail::OpaqueKind::CV_RECT) { + const auto &vec = this_vec.rref(); + const auto roi_mat = preprocess(ctx->inMat(0), vec[list_idx], shape); + copyToOV(roi_mat, input_tensor); + } else if (this_vec.getKind() == cv::detail::OpaqueKind::CV_MAT) { + const auto &vec = this_vec.rref(); + const auto &mat = vec[list_idx]; + copyToOV(mat, input_tensor); + } else { + GAPI_Assert(false && + "OV Backend: Only Rect and Mat types are supported for InferList2"); + } + } + }, + std::bind(callback, std::placeholders::_1, std::placeholders::_2, list_idx) + } // task + ); + } // for + } +}; + } // namespace ov } // namespace gimpl } // namespace cv @@ -858,7 +1316,10 @@ class GOVBackendImpl final: public cv::gapi::GBackend::Priv { } virtual cv::GKernelPackage auxiliaryKernels() const override { - return cv::gapi::kernels< cv::gimpl::ov::Infer >(); + return cv::gapi::kernels< cv::gimpl::ov::Infer + , cv::gimpl::ov::InferROI + , cv::gimpl::ov::InferList + , cv::gimpl::ov::InferList2 >(); } virtual bool controlsMerge() const override { diff --git a/modules/gapi/src/backends/ov/util.hpp b/modules/gapi/src/backends/ov/util.hpp index ea2aeb60a6..896e707c24 100644 --- a/modules/gapi/src/backends/ov/util.hpp +++ b/modules/gapi/src/backends/ov/util.hpp @@ -27,6 +27,8 @@ namespace util { // test suite only. GAPI_EXPORTS std::vector to_ocv(const ::ov::Shape &shape); GAPI_EXPORTS int to_ocv(const ::ov::element::Type &type); +GAPI_EXPORTS void to_ov(const cv::Mat &mat, ::ov::Tensor &tensor); +GAPI_EXPORTS void to_ocv(const ::ov::Tensor &tensor, cv::Mat &mat); }}}} diff --git a/modules/gapi/test/infer/gapi_infer_ov_tests.cpp b/modules/gapi/test/infer/gapi_infer_ov_tests.cpp index ef63f9e8f6..8916b1cdaf 100644 --- a/modules/gapi/test/infer/gapi_infer_ov_tests.cpp +++ b/modules/gapi/test/infer/gapi_infer_ov_tests.cpp @@ -41,20 +41,6 @@ void initDLDTDataPath() static const std::string SUBDIR = "intel/age-gender-recognition-retail-0013/FP32/"; -void copyFromOV(ov::Tensor &tensor, cv::Mat &mat) { - GAPI_Assert(tensor.get_byte_size() == mat.total() * mat.elemSize()); - std::copy_n(reinterpret_cast(tensor.data()), - tensor.get_byte_size(), - mat.ptr()); -} - -void copyToOV(const cv::Mat &mat, ov::Tensor &tensor) { - GAPI_Assert(tensor.get_byte_size() == mat.total() * mat.elemSize()); - std::copy_n(mat.ptr(), - tensor.get_byte_size(), - reinterpret_cast(tensor.data())); -} - // FIXME: taken from the DNN module void normAssert(cv::InputArray ref, cv::InputArray test, const char *comment /*= ""*/, @@ -74,7 +60,7 @@ ov::Core getCore() { // TODO: AGNetGenComp, AGNetTypedComp, AGNetOVComp, AGNetOVCompiled // can be generalized to work with any model and used as parameters for tests. -struct AGNetGenComp { +struct AGNetGenParams { static constexpr const char* tag = "age-gender-generic"; using Params = cv::gapi::ov::Params; @@ -88,19 +74,9 @@ struct AGNetGenComp { const std::string &device) { return {tag, blob_path, device}; } - - static cv::GComputation create() { - cv::GMat in; - GInferInputs inputs; - inputs["data"] = in; - auto outputs = cv::gapi::infer(tag, inputs); - auto age = outputs.at("age_conv3"); - auto gender = outputs.at("prob"); - return cv::GComputation{cv::GIn(in), cv::GOut(age, gender)}; - } }; -struct AGNetTypedComp { +struct AGNetTypedParams { using AGInfo = std::tuple; G_API_NET(AgeGender, , "typed-age-gender"); using Params = cv::gapi::ov::Params; @@ -112,7 +88,9 @@ struct AGNetTypedComp { xml_path, bin_path, device }.cfgOutputLayers({ "age_conv3", "prob" }); } +}; +struct AGNetTypedComp : AGNetTypedParams { static cv::GComputation create() { cv::GMat in; cv::GMat age, gender; @@ -121,30 +99,104 @@ struct AGNetTypedComp { } }; +struct AGNetGenComp : public AGNetGenParams { + static cv::GComputation create() { + cv::GMat in; + GInferInputs inputs; + inputs["data"] = in; + auto outputs = cv::gapi::infer(tag, inputs); + auto age = outputs.at("age_conv3"); + auto gender = outputs.at("prob"); + return cv::GComputation{cv::GIn(in), cv::GOut(age, gender)}; + } +}; + +struct AGNetROIGenComp : AGNetGenParams { + static cv::GComputation create() { + cv::GMat in; + cv::GOpaque roi; + GInferInputs inputs; + inputs["data"] = in; + auto outputs = cv::gapi::infer(tag, roi, inputs); + auto age = outputs.at("age_conv3"); + auto gender = outputs.at("prob"); + return cv::GComputation{cv::GIn(in, roi), cv::GOut(age, gender)}; + } +}; + +struct AGNetListGenComp : AGNetGenParams { + static cv::GComputation create() { + cv::GMat in; + cv::GArray rois; + GInferInputs inputs; + inputs["data"] = in; + auto outputs = cv::gapi::infer(tag, rois, inputs); + auto age = outputs.at("age_conv3"); + auto gender = outputs.at("prob"); + return cv::GComputation{cv::GIn(in, rois), cv::GOut(age, gender)}; + } +}; + +struct AGNetList2GenComp : AGNetGenParams { + static cv::GComputation create() { + cv::GMat in; + cv::GArray rois; + GInferListInputs list; + list["data"] = rois; + auto outputs = cv::gapi::infer2(tag, in, list); + auto age = outputs.at("age_conv3"); + auto gender = outputs.at("prob"); + return cv::GComputation{cv::GIn(in, rois), cv::GOut(age, gender)}; + } +}; + class AGNetOVCompiled { public: AGNetOVCompiled(ov::CompiledModel &&compiled_model) - : m_compiled_model(std::move(compiled_model)) { + : m_compiled_model(std::move(compiled_model)), + m_infer_request(m_compiled_model.create_infer_request()) { + } + + void operator()(const cv::Mat &in_mat, + const cv::Rect &roi, + cv::Mat &age_mat, + cv::Mat &gender_mat) { + // FIXME: W & H could be extracted from model shape + // but it's anyway used only for Age Gender model. + // (Well won't work in case of reshape) + const int W = 62; + const int H = 62; + cv::Mat resized_roi; + cv::resize(in_mat(roi), resized_roi, cv::Size(W, H)); + (*this)(resized_roi, age_mat, gender_mat); + } + + void operator()(const cv::Mat &in_mat, + const std::vector &rois, + std::vector &age_mats, + std::vector &gender_mats) { + for (size_t i = 0; i < rois.size(); ++i) { + (*this)(in_mat, rois[i], age_mats[i], gender_mats[i]); + } } void operator()(const cv::Mat &in_mat, cv::Mat &age_mat, cv::Mat &gender_mat) { - auto infer_request = m_compiled_model.create_infer_request(); - auto input_tensor = infer_request.get_input_tensor(); - copyToOV(in_mat, input_tensor); + auto input_tensor = m_infer_request.get_input_tensor(); + cv::gapi::ov::util::to_ov(in_mat, input_tensor); - infer_request.infer(); + m_infer_request.infer(); - auto age_tensor = infer_request.get_tensor("age_conv3"); + auto age_tensor = m_infer_request.get_tensor("age_conv3"); age_mat.create(cv::gapi::ov::util::to_ocv(age_tensor.get_shape()), cv::gapi::ov::util::to_ocv(age_tensor.get_element_type())); - copyFromOV(age_tensor, age_mat); + cv::gapi::ov::util::to_ocv(age_tensor, age_mat); - auto gender_tensor = infer_request.get_tensor("prob"); + auto gender_tensor = m_infer_request.get_tensor("prob"); gender_mat.create(cv::gapi::ov::util::to_ocv(gender_tensor.get_shape()), cv::gapi::ov::util::to_ocv(gender_tensor.get_element_type())); - copyFromOV(gender_tensor, gender_mat); + cv::gapi::ov::util::to_ocv(gender_tensor, gender_mat); } void export_model(const std::string &outpath) { @@ -155,6 +207,7 @@ public: private: ov::CompiledModel m_compiled_model; + ov::InferRequest m_infer_request; }; struct ImageInputPreproc { @@ -202,19 +255,78 @@ private: std::shared_ptr m_model; }; +struct BaseAgeGenderOV: public ::testing::Test { + BaseAgeGenderOV() { + initDLDTDataPath(); + xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + device = "CPU"; + blob_path = "age-gender-recognition-retail-0013.blob"; + } + + cv::Mat getRandomImage(const cv::Size &sz) { + cv::Mat image(sz, CV_8UC3); + cv::randu(image, 0, 255); + return image; + } + + cv::Mat getRandomTensor(const std::vector &dims, + const int depth) { + cv::Mat tensor(dims, depth); + cv::randu(tensor, -1, 1); + return tensor; + } + + std::string xml_path; + std::string bin_path; + std::string blob_path; + std::string device; + +}; + +struct TestAgeGenderOV : public BaseAgeGenderOV { + cv::Mat ov_age, ov_gender, gapi_age, gapi_gender; + + void validate() { + normAssert(ov_age, gapi_age, "Test age output" ); + normAssert(ov_gender, gapi_gender, "Test gender output"); + } +}; + +struct TestAgeGenderListOV : public BaseAgeGenderOV { + std::vector ov_age, ov_gender, + gapi_age, gapi_gender; + + std::vector roi_list = { + cv::Rect(cv::Point{64, 60}, cv::Size{ 96, 96}), + cv::Rect(cv::Point{50, 32}, cv::Size{128, 160}), + }; + + TestAgeGenderListOV() { + ov_age.resize(roi_list.size()); + ov_gender.resize(roi_list.size()); + gapi_age.resize(roi_list.size()); + gapi_gender.resize(roi_list.size()); + } + + void validate() { + ASSERT_EQ(ov_age.size(), ov_gender.size()); + + ASSERT_EQ(ov_age.size(), gapi_age.size()); + ASSERT_EQ(ov_gender.size(), gapi_gender.size()); + + for (size_t i = 0; i < ov_age.size(); ++i) { + normAssert(ov_age[i], gapi_age[i], "Test age output"); + normAssert(ov_gender[i], gapi_gender[i], "Test gender output"); + } + } +}; + } // anonymous namespace // TODO: Make all of tests below parmetrized to avoid code duplication -TEST(TestAgeGenderOV, InferTypedTensor) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string device = "CPU"; - - cv::Mat in_mat({1, 3, 62, 62}, CV_32F); - cv::randu(in_mat, -1, 1); - cv::Mat ov_age, ov_gender, gapi_age, gapi_gender; - +TEST_F(TestAgeGenderOV, Infer_Tensor) { + const auto in_mat = getRandomTensor({1, 3, 62, 62}, CV_32F); // OpenVINO AGNetOVComp ref(xml_path, bin_path, device); ref.apply(in_mat, ov_age, ov_gender); @@ -226,19 +338,11 @@ TEST(TestAgeGenderOV, InferTypedTensor) { cv::compile_args(cv::gapi::networks(pp))); // Assert - normAssert(ov_age, gapi_age, "Test age output" ); - normAssert(ov_gender, gapi_gender, "Test gender output"); + validate(); } -TEST(TestAgeGenderOV, InferTypedImage) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string device = "CPU"; - - cv::Mat in_mat(300, 300, CV_8UC3); - cv::randu(in_mat, 0, 255); - cv::Mat ov_age, ov_gender, gapi_age, gapi_gender; +TEST_F(TestAgeGenderOV, Infer_Image) { + const auto in_mat = getRandomImage({300, 300}); // OpenVINO AGNetOVComp ref(xml_path, bin_path, device); @@ -252,19 +356,11 @@ TEST(TestAgeGenderOV, InferTypedImage) { cv::compile_args(cv::gapi::networks(pp))); // Assert - normAssert(ov_age, gapi_age, "Test age output" ); - normAssert(ov_gender, gapi_gender, "Test gender output"); + validate(); } -TEST(TestAgeGenderOV, InferGenericTensor) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string device = "CPU"; - - cv::Mat in_mat({1, 3, 62, 62}, CV_32F); - cv::randu(in_mat, -1, 1); - cv::Mat ov_age, ov_gender, gapi_age, gapi_gender; +TEST_F(TestAgeGenderOV, InferGeneric_Tensor) { + const auto in_mat = getRandomTensor({1, 3, 62, 62}, CV_32F); // OpenVINO AGNetOVComp ref(xml_path, bin_path, device); @@ -277,19 +373,11 @@ TEST(TestAgeGenderOV, InferGenericTensor) { cv::compile_args(cv::gapi::networks(pp))); // Assert - normAssert(ov_age, gapi_age, "Test age output" ); - normAssert(ov_gender, gapi_gender, "Test gender output"); + validate(); } -TEST(TestAgeGenderOV, InferGenericImage) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string device = "CPU"; - - cv::Mat in_mat(300, 300, CV_8UC3); - cv::randu(in_mat, 0, 255); - cv::Mat ov_age, ov_gender, gapi_age, gapi_gender; +TEST_F(TestAgeGenderOV, InferGenericImage) { + const auto in_mat = getRandomImage({300, 300}); // OpenVINO AGNetOVComp ref(xml_path, bin_path, device); @@ -303,20 +391,11 @@ TEST(TestAgeGenderOV, InferGenericImage) { cv::compile_args(cv::gapi::networks(pp))); // Assert - normAssert(ov_age, gapi_age, "Test age output" ); - normAssert(ov_gender, gapi_gender, "Test gender output"); + validate(); } -TEST(TestAgeGenderOV, InferGenericImageBlob) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string blob_path = "age-gender-recognition-retail-0013.blob"; - const std::string device = "CPU"; - - cv::Mat in_mat(300, 300, CV_8UC3); - cv::randu(in_mat, 0, 255); - cv::Mat ov_age, ov_gender, gapi_age, gapi_gender; +TEST_F(TestAgeGenderOV, InferGeneric_ImageBlob) { + const auto in_mat = getRandomImage({300, 300}); // OpenVINO AGNetOVComp ref(xml_path, bin_path, device); @@ -333,20 +412,11 @@ TEST(TestAgeGenderOV, InferGenericImageBlob) { cv::compile_args(cv::gapi::networks(pp))); // Assert - normAssert(ov_age, gapi_age, "Test age output" ); - normAssert(ov_gender, gapi_gender, "Test gender output"); + validate(); } -TEST(TestAgeGenderOV, InferGenericTensorBlob) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string blob_path = "age-gender-recognition-retail-0013.blob"; - const std::string device = "CPU"; - - cv::Mat in_mat({1, 3, 62, 62}, CV_32F); - cv::randu(in_mat, -1, 1); - cv::Mat ov_age, ov_gender, gapi_age, gapi_gender; +TEST_F(TestAgeGenderOV, InferGeneric_TensorBlob) { + const auto in_mat = getRandomTensor({1, 3, 62, 62}, CV_32F); // OpenVINO AGNetOVComp ref(xml_path, bin_path, device); @@ -361,19 +431,11 @@ TEST(TestAgeGenderOV, InferGenericTensorBlob) { cv::compile_args(cv::gapi::networks(pp))); // Assert - normAssert(ov_age, gapi_age, "Test age output" ); - normAssert(ov_gender, gapi_gender, "Test gender output"); + validate(); } -TEST(TestAgeGenderOV, InferBothOutputsFP16) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string device = "CPU"; - - cv::Mat in_mat({1, 3, 62, 62}, CV_32F); - cv::randu(in_mat, -1, 1); - cv::Mat ov_age, ov_gender, gapi_age, gapi_gender; +TEST_F(TestAgeGenderOV, InferGeneric_BothOutputsFP16) { + const auto in_mat = getRandomTensor({1, 3, 62, 62}, CV_32F); // OpenVINO AGNetOVComp ref(xml_path, bin_path, device); @@ -392,19 +454,11 @@ TEST(TestAgeGenderOV, InferBothOutputsFP16) { cv::compile_args(cv::gapi::networks(pp))); // Assert - normAssert(ov_age, gapi_age, "Test age output" ); - normAssert(ov_gender, gapi_gender, "Test gender output"); + validate(); } -TEST(TestAgeGenderOV, InferOneOutputFP16) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string device = "CPU"; - - cv::Mat in_mat({1, 3, 62, 62}, CV_32F); - cv::randu(in_mat, -1, 1); - cv::Mat ov_age, ov_gender, gapi_age, gapi_gender; +TEST_F(TestAgeGenderOV, InferGeneric_OneOutputFP16) { + const auto in_mat = getRandomTensor({1, 3, 62, 62}, CV_32F); // OpenVINO const std::string fp16_output_name = "prob"; @@ -423,17 +477,10 @@ TEST(TestAgeGenderOV, InferOneOutputFP16) { cv::compile_args(cv::gapi::networks(pp))); // Assert - normAssert(ov_age, gapi_age, "Test age output" ); - normAssert(ov_gender, gapi_gender, "Test gender output"); + validate(); } -TEST(TestAgeGenderOV, ThrowCfgOutputPrecForBlob) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string blob_path = "age-gender-recognition-retail-0013.blob"; - const std::string device = "CPU"; - +TEST_F(TestAgeGenderOV, InferGeneric_ThrowCfgOutputPrecForBlob) { // OpenVINO (Just for blob compilation) AGNetOVComp ref(xml_path, bin_path, device); auto cc_ref = ref.compile(); @@ -446,12 +493,7 @@ TEST(TestAgeGenderOV, ThrowCfgOutputPrecForBlob) { EXPECT_ANY_THROW(pp.cfgOutputTensorPrecision(CV_16F)); } -TEST(TestAgeGenderOV, ThrowInvalidConfigIR) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string device = "CPU"; - +TEST_F(TestAgeGenderOV, InferGeneric_ThrowInvalidConfigIR) { // G-API auto comp = AGNetGenComp::create(); auto pp = AGNetGenComp::params(xml_path, bin_path, device); @@ -461,13 +503,7 @@ TEST(TestAgeGenderOV, ThrowInvalidConfigIR) { cv::compile_args(cv::gapi::networks(pp)))); } -TEST(TestAgeGenderOV, ThrowInvalidConfigBlob) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string blob_path = "age-gender-recognition-retail-0013.blob"; - const std::string device = "CPU"; - +TEST_F(TestAgeGenderOV, InferGeneric_ThrowInvalidConfigBlob) { // OpenVINO (Just for blob compilation) AGNetOVComp ref(xml_path, bin_path, device); auto cc_ref = ref.compile(); @@ -482,16 +518,8 @@ TEST(TestAgeGenderOV, ThrowInvalidConfigBlob) { cv::compile_args(cv::gapi::networks(pp)))); } -TEST(TestAgeGenderOV, ThrowInvalidImageLayout) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string device = "CPU"; - - // NB: This mat may only have "NHWC" layout. - cv::Mat in_mat(300, 300, CV_8UC3); - cv::randu(in_mat, 0, 255); - cv::Mat gender, gapi_age, gapi_gender; +TEST_F(TestAgeGenderOV, Infer_ThrowInvalidImageLayout) { + const auto in_mat = getRandomImage({300, 300}); auto comp = AGNetTypedComp::create(); auto pp = AGNetTypedComp::params(xml_path, bin_path, device); @@ -501,15 +529,8 @@ TEST(TestAgeGenderOV, ThrowInvalidImageLayout) { cv::compile_args(cv::gapi::networks(pp)))); } -TEST(TestAgeGenderOV, InferTensorWithPreproc) { - initDLDTDataPath(); - const std::string xml_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); - const std::string bin_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); - const std::string device = "CPU"; - - cv::Mat in_mat({1, 240, 320, 3}, CV_32F); - cv::randu(in_mat, -1, 1); - cv::Mat ov_age, ov_gender, gapi_age, gapi_gender; +TEST_F(TestAgeGenderOV, Infer_TensorWithPreproc) { + const auto in_mat = getRandomTensor({1, 240, 320, 3}, CV_32F); // OpenVINO AGNetOVComp ref(xml_path, bin_path, device); @@ -531,8 +552,112 @@ TEST(TestAgeGenderOV, InferTensorWithPreproc) { cv::compile_args(cv::gapi::networks(pp))); // Assert - normAssert(ov_age, gapi_age, "Test age output" ); - normAssert(ov_gender, gapi_gender, "Test gender output"); + validate(); +} + +TEST_F(TestAgeGenderOV, InferROIGeneric_Image) { + const auto in_mat = getRandomImage({300, 300}); + cv::Rect roi(cv::Rect(cv::Point{64, 60}, cv::Size{96, 96})); + + // OpenVINO + AGNetOVComp ref(xml_path, bin_path, device); + ref.cfgPrePostProcessing([](ov::preprocess::PrePostProcessor &ppp) { + ppp.input().tensor().set_element_type(ov::element::u8); + ppp.input().tensor().set_layout("NHWC"); + }); + ref.compile()(in_mat, roi, ov_age, ov_gender); + + // G-API + auto comp = AGNetROIGenComp::create(); + auto pp = AGNetROIGenComp::params(xml_path, bin_path, device); + + comp.apply(cv::gin(in_mat, roi), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp))); + + // Assert + validate(); +} + +TEST_F(TestAgeGenderOV, InferROIGeneric_ThrowIncorrectLayout) { + const auto in_mat = getRandomImage({300, 300}); + cv::Rect roi(cv::Rect(cv::Point{64, 60}, cv::Size{96, 96})); + + // G-API + auto comp = AGNetROIGenComp::create(); + auto pp = AGNetROIGenComp::params(xml_path, bin_path, device); + + pp.cfgInputTensorLayout("NCHW"); + EXPECT_ANY_THROW(comp.apply(cv::gin(in_mat, roi), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp)))); +} + +TEST_F(TestAgeGenderOV, InferROIGeneric_ThrowTensorInput) { + const auto in_mat = getRandomTensor({1, 3, 62, 62}, CV_32F); + cv::Rect roi(cv::Rect(cv::Point{64, 60}, cv::Size{96, 96})); + + // G-API + auto comp = AGNetROIGenComp::create(); + auto pp = AGNetROIGenComp::params(xml_path, bin_path, device); + + EXPECT_ANY_THROW(comp.apply(cv::gin(in_mat, roi), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp)))); +} + +TEST_F(TestAgeGenderOV, InferROIGeneric_ThrowExplicitResize) { + const auto in_mat = getRandomImage({300, 300}); + cv::Rect roi(cv::Rect(cv::Point{64, 60}, cv::Size{96, 96})); + + // G-API + auto comp = AGNetROIGenComp::create(); + auto pp = AGNetROIGenComp::params(xml_path, bin_path, device); + + pp.cfgResize(cv::INTER_LINEAR); + EXPECT_ANY_THROW(comp.apply(cv::gin(in_mat, roi), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp)))); +} + +TEST_F(TestAgeGenderListOV, InferListGeneric_Image) { + const auto in_mat = getRandomImage({300, 300}); + + // OpenVINO + AGNetOVComp ref(xml_path, bin_path, device); + ref.cfgPrePostProcessing([](ov::preprocess::PrePostProcessor &ppp) { + ppp.input().tensor().set_element_type(ov::element::u8); + ppp.input().tensor().set_layout("NHWC"); + }); + ref.compile()(in_mat, roi_list, ov_age, ov_gender); + + // G-API + auto comp = AGNetListGenComp::create(); + auto pp = AGNetListGenComp::params(xml_path, bin_path, device); + + comp.apply(cv::gin(in_mat, roi_list), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp))); + + // Assert + validate(); +} + +TEST_F(TestAgeGenderListOV, InferList2Generic_Image) { + const auto in_mat = getRandomImage({300, 300}); + + // OpenVINO + AGNetOVComp ref(xml_path, bin_path, device); + ref.cfgPrePostProcessing([](ov::preprocess::PrePostProcessor &ppp) { + ppp.input().tensor().set_element_type(ov::element::u8); + ppp.input().tensor().set_layout("NHWC"); + }); + ref.compile()(in_mat, roi_list, ov_age, ov_gender); + + // G-API + auto comp = AGNetList2GenComp::create(); + auto pp = AGNetList2GenComp::params(xml_path, bin_path, device); + + comp.apply(cv::gin(in_mat, roi_list), cv::gout(gapi_age, gapi_gender), + cv::compile_args(cv::gapi::networks(pp))); + + // Assert + validate(); } } // namespace opencv_test From 805946baaf43a0a862cb678b4c1841ab01ee697a Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 20 Jun 2023 14:10:08 +0300 Subject: [PATCH 023/105] pre: OpenCV 3.4.20 (version++) --- .../cross_referencing/tutorial_cross_referencing.markdown | 4 ++-- modules/core/include/opencv2/core/version.hpp | 4 ++-- modules/python/package/setup.py | 2 +- platforms/android/build_sdk.py | 2 +- platforms/android/service/readme.txt | 2 +- platforms/maven/opencv-it/pom.xml | 2 +- platforms/maven/opencv/pom.xml | 2 +- platforms/maven/pom.xml | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown index daa7c396cd..ed13c3e030 100644 --- a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown +++ b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown @@ -39,14 +39,14 @@ Open your Doxyfile using your favorite text editor and search for the key `TAGFILES`. Change it as follows: @code -TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.19 +TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.20 @endcode If you had other definitions already, you can append the line using a `\`: @code TAGFILES = ./docs/doxygen-tags/libstdc++.tag=https://gcc.gnu.org/onlinedocs/libstdc++/latest-doxygen \ - ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.19 + ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/3.4.20 @endcode Doxygen can now use the information from the tag file to link to the OpenCV diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index e355498735..e4418b7857 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -7,8 +7,8 @@ #define CV_VERSION_MAJOR 3 #define CV_VERSION_MINOR 4 -#define CV_VERSION_REVISION 19 -#define CV_VERSION_STATUS "-dev" +#define CV_VERSION_REVISION 20 +#define CV_VERSION_STATUS "-pre" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) diff --git a/modules/python/package/setup.py b/modules/python/package/setup.py index 1356931720..d91cb1d51a 100644 --- a/modules/python/package/setup.py +++ b/modules/python/package/setup.py @@ -9,7 +9,7 @@ def main(): os.chdir(SCRIPT_DIR) package_name = 'opencv' - package_version = os.environ.get('OPENCV_VERSION', '3.4.19') # TODO + package_version = os.environ.get('OPENCV_VERSION', '3.4.20') # TODO long_description = 'Open Source Computer Vision Library Python bindings' # TODO diff --git a/platforms/android/build_sdk.py b/platforms/android/build_sdk.py index 3a9618b835..62799a91a1 100755 --- a/platforms/android/build_sdk.py +++ b/platforms/android/build_sdk.py @@ -269,7 +269,7 @@ class Builder: # Add extra data apkxmldest = check_dir(os.path.join(apkdest, "res", "xml"), create=True) apklibdest = check_dir(os.path.join(apkdest, "libs", abi.name), create=True) - for ver, d in self.extra_packs + [("3.4.19", os.path.join(self.libdest, "lib"))]: + for ver, d in self.extra_packs + [("3.4.20", os.path.join(self.libdest, "lib"))]: r = ET.Element("library", attrib={"version": ver}) log.info("Adding libraries from %s", d) diff --git a/platforms/android/service/readme.txt b/platforms/android/service/readme.txt index 11e9f0bae8..fec5c6746a 100644 --- a/platforms/android/service/readme.txt +++ b/platforms/android/service/readme.txt @@ -12,7 +12,7 @@ manually using adb tool: adb install /apk/OpenCV__Manager__.apk -Example: OpenCV_3.4.19-dev_Manager_3.49_armeabi-v7a.apk +Example: OpenCV_3.4.20-dev_Manager_3.49_armeabi-v7a.apk Use the list of platforms below to determine proper OpenCV Manager package for your device: diff --git a/platforms/maven/opencv-it/pom.xml b/platforms/maven/opencv-it/pom.xml index befb75fed5..5955905b9b 100644 --- a/platforms/maven/opencv-it/pom.xml +++ b/platforms/maven/opencv-it/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 3.4.19 + 3.4.20 org.opencv opencv-it diff --git a/platforms/maven/opencv/pom.xml b/platforms/maven/opencv/pom.xml index 0ff0c6fe2f..354d15d334 100644 --- a/platforms/maven/opencv/pom.xml +++ b/platforms/maven/opencv/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 3.4.19 + 3.4.20 org.opencv opencv diff --git a/platforms/maven/pom.xml b/platforms/maven/pom.xml index 19ba7d07b3..04aa302c41 100644 --- a/platforms/maven/pom.xml +++ b/platforms/maven/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.opencv opencv-parent - 3.4.19 + 3.4.20 pom OpenCV Parent POM From 51702ffd92448dc40b56c8040525ec0d3e353f21 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 20 Jun 2023 15:52:57 +0300 Subject: [PATCH 024/105] pre: OpenCV 4.8.0 (version++) --- .../cross_referencing/tutorial_cross_referencing.markdown | 4 ++-- modules/core/include/opencv2/core/version.hpp | 4 ++-- modules/dnn/include/opencv2/dnn/version.hpp | 2 +- modules/python/package/setup.py | 2 +- platforms/maven/opencv-it/pom.xml | 2 +- platforms/maven/opencv/pom.xml | 2 +- platforms/maven/pom.xml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown index b77891cee9..ec605c8e45 100644 --- a/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown +++ b/doc/tutorials/introduction/cross_referencing/tutorial_cross_referencing.markdown @@ -46,14 +46,14 @@ Open your Doxyfile using your favorite text editor and search for the key `TAGFILES`. Change it as follows: @code -TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.7.0 +TAGFILES = ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.8.0 @endcode If you had other definitions already, you can append the line using a `\`: @code TAGFILES = ./docs/doxygen-tags/libstdc++.tag=https://gcc.gnu.org/onlinedocs/libstdc++/latest-doxygen \ - ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.7.0 + ./docs/doxygen-tags/opencv.tag=http://docs.opencv.org/4.8.0 @endcode Doxygen can now use the information from the tag file to link to the OpenCV diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index f158eb842c..2dec11e20c 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -6,9 +6,9 @@ #define OPENCV_VERSION_HPP #define CV_VERSION_MAJOR 4 -#define CV_VERSION_MINOR 7 +#define CV_VERSION_MINOR 8 #define CV_VERSION_REVISION 0 -#define CV_VERSION_STATUS "-dev" +#define CV_VERSION_STATUS "-pre" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) diff --git a/modules/dnn/include/opencv2/dnn/version.hpp b/modules/dnn/include/opencv2/dnn/version.hpp index 93b5ff3158..b93622def9 100644 --- a/modules/dnn/include/opencv2/dnn/version.hpp +++ b/modules/dnn/include/opencv2/dnn/version.hpp @@ -6,7 +6,7 @@ #define OPENCV_DNN_VERSION_HPP /// Use with major OpenCV version only. -#define OPENCV_DNN_API_VERSION 20221220 +#define OPENCV_DNN_API_VERSION 20230620 #if !defined CV_DOXYGEN && !defined CV_STATIC_ANALYSIS && !defined CV_DNN_DONT_ADD_INLINE_NS #define CV__DNN_INLINE_NS __CV_CAT(dnn4_v, OPENCV_DNN_API_VERSION) diff --git a/modules/python/package/setup.py b/modules/python/package/setup.py index d245b9da7b..11b204d603 100644 --- a/modules/python/package/setup.py +++ b/modules/python/package/setup.py @@ -19,7 +19,7 @@ def main(): os.chdir(SCRIPT_DIR) package_name = 'opencv' - package_version = os.environ.get('OPENCV_VERSION', '4.7.0') # TODO + package_version = os.environ.get('OPENCV_VERSION', '4.8.0') # TODO long_description = 'Open Source Computer Vision Library Python bindings' # TODO diff --git a/platforms/maven/opencv-it/pom.xml b/platforms/maven/opencv-it/pom.xml index f4699df1bf..8e987002d8 100644 --- a/platforms/maven/opencv-it/pom.xml +++ b/platforms/maven/opencv-it/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 4.7.0 + 4.8.0 org.opencv opencv-it diff --git a/platforms/maven/opencv/pom.xml b/platforms/maven/opencv/pom.xml index bfbf2eef54..2aba1156f3 100644 --- a/platforms/maven/opencv/pom.xml +++ b/platforms/maven/opencv/pom.xml @@ -4,7 +4,7 @@ org.opencv opencv-parent - 4.7.0 + 4.8.0 org.opencv opencv diff --git a/platforms/maven/pom.xml b/platforms/maven/pom.xml index 73d5302c22..b6b57ed916 100644 --- a/platforms/maven/pom.xml +++ b/platforms/maven/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.opencv opencv-parent - 4.7.0 + 4.8.0 pom OpenCV Parent POM From 06b40aef91de569ba11d88aea2b8d0dfc1dcd2d6 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Tue, 20 Jun 2023 16:31:55 +0300 Subject: [PATCH 025/105] fix: AST nodes required usage imports --- modules/python/src2/typing_stubs_generation/generation.py | 2 -- .../python/src2/typing_stubs_generation/nodes/type_node.py | 5 +++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index 5b3a2ebe00..b830408b26 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -759,8 +759,6 @@ def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None: output_stream.write(f' "{alias_name}",\n') output_stream.write("]\n\n") - # HACK: force add cv2.mat_wrapper import to handle MatLike alias - required_imports.add("import cv2.mat_wrapper") _write_required_imports(required_imports, output_stream) # Add type checking time definitions as generated __init__.py content diff --git a/modules/python/src2/typing_stubs_generation/nodes/type_node.py b/modules/python/src2/typing_stubs_generation/nodes/type_node.py index f7018938c1..16a98aa415 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/type_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/type_node.py @@ -561,13 +561,14 @@ class ContainerTypeNode(AggregatedTypeNode): @property def required_definition_imports(self) -> Generator[str, None, None]: yield "import typing" - return super().required_definition_imports + yield from super().required_definition_imports @property def required_usage_imports(self) -> Generator[str, None, None]: if TypeNode.compatible_to_runtime_usage: yield "import typing" - return super().required_usage_imports + yield from super().required_usage_imports + @abc.abstractproperty def type_format(self) -> str: pass From f20edba925f88cacbaf3af3d6a82c36fe4e9fffd Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Tue, 20 Jun 2023 20:05:58 +0300 Subject: [PATCH 026/105] fix: conditionally define generic NumPy NDArray alias --- .../typing_stubs_generation/generation.py | 58 +++++--- .../nodes/type_node.py | 139 +++++++++++++++--- .../predefined_types.py | 27 +++- 3 files changed, 181 insertions(+), 43 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index b830408b26..55d3544730 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -15,7 +15,8 @@ from .nodes import (ASTNode, ASTNodeType, NamespaceNode, ClassNode, FunctionNode EnumerationNode, ConstantNode) from .nodes.type_node import (TypeNode, AliasTypeNode, AliasRefTypeNode, - AggregatedTypeNode, ASTNodeTypeNode) + AggregatedTypeNode, ASTNodeTypeNode, + ConditionalAliasTypeNode, PrimitiveTypeNode) def generate_typing_stubs(root: NamespaceNode, output_path: Path): @@ -682,28 +683,37 @@ def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None: f"Provided type node '{type_node.ctype_name}' is not an aggregated type" for item in filter(lambda i: isinstance(i, AliasRefTypeNode), type_node): - register_alias(PREDEFINED_TYPES[item.ctype_name]) # type: ignore + type_node = PREDEFINED_TYPES[item.ctype_name] + if isinstance(type_node, AliasTypeNode): + register_alias(type_node) + elif isinstance(type_node, ConditionalAliasTypeNode): + conditional_type_nodes[type_node.ctype_name] = type_node - def create_alias_for_enum_node(enum_node: ASTNode) -> AliasTypeNode: - """Create int alias corresponding to the given enum node. + def create_alias_for_enum_node(enum_node_alias: AliasTypeNode) -> ConditionalAliasTypeNode: + """Create conditional int alias corresponding to the given enum node. Args: - enum_node (ASTNodeTypeNode): Enumeration node to create int alias for. + enum_node (AliasTypeNode): Enumeration node to create conditional + int alias for. Returns: - AliasTypeNode: int alias node with same export name as enum. + ConditionalAliasTypeNode: conditional int alias node with same + export name as enum. """ + enum_node = enum_node_alias.ast_node assert enum_node.node_type == ASTNodeType.Enumeration, \ f"{enum_node} has wrong node type. Expected type: Enumeration." enum_export_name, enum_module_name = get_enum_module_and_export_name( enum_node ) - enum_full_export_name = f"{enum_module_name}.{enum_export_name}" - alias_node = AliasTypeNode.int_(enum_full_export_name, - enum_export_name) - type_checking_time_definitions.add(alias_node) - return alias_node + return ConditionalAliasTypeNode( + enum_export_name, + "typing.TYPE_CHECKING", + positive_branch_type=enum_node_alias, + negative_branch_type=PrimitiveTypeNode.int_(enum_export_name), + condition_required_imports=("import typing", ) + ) def register_alias(alias_node: AliasTypeNode) -> None: typename = alias_node.typename @@ -726,11 +736,15 @@ def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None: continue if item.ast_node.node_type != ASTNodeType.Enumeration: continue - alias_node.value.items[i] = create_alias_for_enum_node(item.ast_node) + enum_node = create_alias_for_enum_node(item) + alias_node.value.items[i] = enum_node + conditional_type_nodes[enum_node.ctype_name] = enum_node if isinstance(alias_node.value, ASTNodeTypeNode) \ and alias_node.value.ast_node == ASTNodeType.Enumeration: - alias_node.value = create_alias_for_enum_node(alias_node.ast_node) + enum_node = create_alias_for_enum_node(alias_node.ast_node) + conditional_type_nodes[enum_node.ctype_name] = enum_node + return # Strip module prefix from aliased types aliases[typename] = alias_node.value.full_typename.replace( @@ -744,7 +758,7 @@ def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None: required_imports: Set[str] = set() aliases: Dict[str, str] = {} - type_checking_time_definitions: Set[AliasTypeNode] = set() + conditional_type_nodes: Dict[str, ConditionalAliasTypeNode] = {} # Resolve each node and register aliases TypeNode.compatible_to_runtime_usage = True @@ -752,6 +766,12 @@ def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None: node.resolve(root) if isinstance(node, AliasTypeNode): register_alias(node) + elif isinstance(node, ConditionalAliasTypeNode): + conditional_type_nodes[node.ctype_name] = node + + for node in conditional_type_nodes.values(): + for required_import in node.required_definition_imports: + required_imports.add(required_import) output_stream = StringIO() output_stream.write("__all__ = [\n") @@ -762,12 +782,10 @@ def _generate_typing_module(root: NamespaceNode, output_path: Path) -> None: _write_required_imports(required_imports, output_stream) # Add type checking time definitions as generated __init__.py content - for alias in type_checking_time_definitions: - output_stream.write("if typing.TYPE_CHECKING:\n ") - output_stream.write(f"{alias.typename} = {alias.ctype_name}\nelse:\n") - output_stream.write(f" {alias.typename} = {alias.value.ctype_name}\n") - if type_checking_time_definitions: - output_stream.write("\n\n") + for _, type_node in conditional_type_nodes.items(): + output_stream.write(f"if {type_node.condition}:\n ") + output_stream.write(f"{type_node.typename} = {type_node.positive_branch_type.full_typename}\nelse:\n") + output_stream.write(f" {type_node.typename} = {type_node.negative_branch_type.full_typename}\n\n\n") for alias_name, alias_type in aliases.items(): output_stream.write(f"{alias_name} = {alias_type}\n") diff --git a/modules/python/src2/typing_stubs_generation/nodes/type_node.py b/modules/python/src2/typing_stubs_generation/nodes/type_node.py index 16a98aa415..912adc6954 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/type_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/type_node.py @@ -307,14 +307,31 @@ class AliasTypeNode(TypeNode): return cls(ctype_name, PrimitiveTypeNode.float_(), export_name, doc) @classmethod - def array_(cls, ctype_name: str, shape: Optional[Tuple[int, ...]], - dtype: Optional[str] = None, export_name: Optional[str] = None, - doc: Optional[str] = None): + def array_ref_(cls, ctype_name: str, array_ref_name: str, + shape: Optional[Tuple[int, ...]], + dtype: Optional[str] = None, + export_name: Optional[str] = None, + doc: Optional[str] = None): + """Create alias to array reference alias `array_ref_name`. + + This is required to preserve backward compatibility with Python < 3.9 + and NumPy 1.20, when NumPy module introduces generics support. + + Args: + ctype_name (str): Name of the alias. + array_ref_name (str): Name of the conditional array alias. + shape (Optional[Tuple[int, ...]]): Array shape. + dtype (Optional[str], optional): Array type. Defaults to None. + export_name (Optional[str], optional): Alias export name. + Defaults to None. + doc (Optional[str], optional): Documentation string for alias. + Defaults to None. + """ if doc is None: - doc = "Shape: " + str(shape) + doc = f"NDArray(shape={shape}, dtype={dtype})" else: - doc += ". Shape: " + str(shape) - return cls(ctype_name, NDArrayTypeNode(ctype_name, shape, dtype), + doc += f". NDArray(shape={shape}, dtype={dtype})" + return cls(ctype_name, AliasRefTypeNode(array_ref_name), export_name, doc) @classmethod @@ -376,24 +393,112 @@ class AliasTypeNode(TypeNode): export_name, doc) -class NDArrayTypeNode(TypeNode): - """Type node representing NumPy ndarray. +class ConditionalAliasTypeNode(TypeNode): + """Type node representing an alias protected by condition checked in runtime. + Example: + ```python + if numpy.lib.NumpyVersion(numpy.__version__) > "1.20.0" and sys.version_info >= (3, 9) + NumPyArray = numpy.ndarray[typing.Any, numpy.dtype[numpy.generic]] + else: + NumPyArray = numpy.ndarray + ``` + is defined as follows: + ```python + + ConditionalAliasTypeNode( + "NumPyArray", + 'numpy.lib.NumpyVersion(numpy.__version__) > "1.20.0" and sys.version_info >= (3, 9)', + NDArrayTypeNode("NumPyArray"), + NDArrayTypeNode("NumPyArray", use_numpy_generics=False), + condition_required_imports=("import numpy", "import sys") + ) + ``` """ - def __init__(self, ctype_name: str, shape: Optional[Tuple[int, ...]] = None, - dtype: Optional[str] = None) -> None: + def __init__(self, ctype_name: str, condition: str, + positive_branch_type: TypeNode, + negative_branch_type: TypeNode, + export_name: Optional[str] = None, + condition_required_imports: Sequence[str] = ()) -> None: super().__init__(ctype_name) - self.shape = shape - self.dtype = dtype + self.condition = condition + self.positive_branch_type = positive_branch_type + self.positive_branch_type.ctype_name = self.ctype_name + self.negative_branch_type = negative_branch_type + self.negative_branch_type.ctype_name = self.ctype_name + self._export_name = export_name + self._condition_required_imports = condition_required_imports @property def typename(self) -> str: - return "numpy.ndarray[{shape}, numpy.dtype[{dtype}]]".format( - # NOTE: Shape is not fully supported yet - # shape=self.shape if self.shape is not None else "typing.Any", - shape="typing.Any", - dtype=self.dtype if self.dtype is not None else "numpy.generic" + if self._export_name is not None: + return self._export_name + return self.ctype_name + + @property + def full_typename(self) -> str: + return "cv2.typing." + self.typename + + @property + def required_definition_imports(self) -> Generator[str, None, None]: + yield from self.positive_branch_type.required_usage_imports + yield from self.negative_branch_type.required_usage_imports + yield from self._condition_required_imports + + @property + def required_usage_imports(self) -> Generator[str, None, None]: + yield "import cv2.typing" + + @property + def is_resolved(self) -> bool: + return self.positive_branch_type.is_resolved \ + and self.negative_branch_type.is_resolved + + def resolve(self, root: ASTNode): + try: + self.positive_branch_type.resolve(root) + self.negative_branch_type.resolve(root) + except TypeResolutionError as e: + raise TypeResolutionError( + 'Failed to resolve alias "{}" exposed as "{}"'.format( + self.ctype_name, self.typename + ) + ) from e + + @classmethod + def numpy_array_(cls, ctype_name: str, export_name: Optional[str] = None, + shape: Optional[Tuple[int, ...]] = None, + dtype: Optional[str] = None): + return cls( + ctype_name, + ('numpy.lib.NumpyVersion(numpy.__version__) > "1.20.0" ' + 'and sys.version_info >= (3, 9)'), + NDArrayTypeNode(ctype_name, shape, dtype), + NDArrayTypeNode(ctype_name, shape, dtype, + use_numpy_generics=False), + condition_required_imports=("import numpy", "import sys") ) + +class NDArrayTypeNode(TypeNode): + """Type node representing NumPy ndarray. + """ + def __init__(self, ctype_name: str, + shape: Optional[Tuple[int, ...]] = None, + dtype: Optional[str] = None, + use_numpy_generics: bool = True) -> None: + super().__init__(ctype_name) + self.shape = shape + self.dtype = dtype + self._use_numpy_generics = use_numpy_generics + + @property + def typename(self) -> str: + if self._use_numpy_generics: + # NOTE: Shape is not fully supported yet + dtype = self.dtype if self.dtype is not None else "numpy.generic" + return f"numpy.ndarray[typing.Any, numpy.dtype[{dtype}]]" + return "numpy.ndarray" + @property def required_usage_imports(self) -> Generator[str, None, None]: yield "import numpy" diff --git a/modules/python/src2/typing_stubs_generation/predefined_types.py b/modules/python/src2/typing_stubs_generation/predefined_types.py index 842ab3be6e..fe9a37a45e 100644 --- a/modules/python/src2/typing_stubs_generation/predefined_types.py +++ b/modules/python/src2/typing_stubs_generation/predefined_types.py @@ -1,7 +1,7 @@ from .nodes.type_node import ( AliasTypeNode, AliasRefTypeNode, PrimitiveTypeNode, ASTNodeTypeNode, NDArrayTypeNode, NoneTypeNode, SequenceTypeNode, - TupleTypeNode, UnionTypeNode, AnyTypeNode + TupleTypeNode, UnionTypeNode, AnyTypeNode, ConditionalAliasTypeNode ) # Set of predefined types used to cover cases when library doesn't @@ -30,12 +30,15 @@ _PREDEFINED_TYPES = ( PrimitiveTypeNode.str_("char"), PrimitiveTypeNode.str_("String"), PrimitiveTypeNode.str_("c_string"), + ConditionalAliasTypeNode.numpy_array_("NumPyArrayGeneric"), + ConditionalAliasTypeNode.numpy_array_("NumPyArrayFloat32", dtype="numpy.float32"), + ConditionalAliasTypeNode.numpy_array_("NumPyArrayFloat64", dtype="numpy.float64"), NoneTypeNode("void"), AliasTypeNode.int_("void*", "IntPointer", "Represents an arbitrary pointer"), AliasTypeNode.union_( "Mat", items=(ASTNodeTypeNode("Mat", module_name="cv2.mat_wrapper"), - NDArrayTypeNode("Mat")), + AliasRefTypeNode("NumPyArrayGeneric")), export_name="MatLike" ), AliasTypeNode.sequence_("MatShape", PrimitiveTypeNode.int_()), @@ -137,10 +140,22 @@ _PREDEFINED_TYPES = ( ASTNodeTypeNode("gapi.wip.draw.Mosaic"), ASTNodeTypeNode("gapi.wip.draw.Poly"))), SequenceTypeNode("Prims", AliasRefTypeNode("Prim")), - AliasTypeNode.array_("Matx33f", (3, 3), "numpy.float32"), - AliasTypeNode.array_("Matx33d", (3, 3), "numpy.float64"), - AliasTypeNode.array_("Matx44f", (4, 4), "numpy.float32"), - AliasTypeNode.array_("Matx44d", (4, 4), "numpy.float64"), + AliasTypeNode.array_ref_("Matx33f", + array_ref_name="NumPyArrayFloat32", + shape=(3, 3), + dtype="numpy.float32"), + AliasTypeNode.array_ref_("Matx33d", + array_ref_name="NumPyArrayFloat64", + shape=(3, 3), + dtype="numpy.float64"), + AliasTypeNode.array_ref_("Matx44f", + array_ref_name="NumPyArrayFloat32", + shape=(4, 4), + dtype="numpy.float32"), + AliasTypeNode.array_ref_("Matx44d", + array_ref_name="NumPyArrayFloat64", + shape=(4, 4), + dtype="numpy.float64"), NDArrayTypeNode("vector", dtype="numpy.uint8"), NDArrayTypeNode("vector_uchar", dtype="numpy.uint8"), TupleTypeNode("GMat2", items=(ASTNodeTypeNode("GMat"), From 9eaa7bd566863f86776fbbc5635edfa3f399d812 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 20 Jun 2023 15:15:18 +0300 Subject: [PATCH 027/105] Document parameters of multi-dimentional reshape. --- modules/core/include/opencv2/core/mat.hpp | 25 +++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/mat.hpp b/modules/core/include/opencv2/core/mat.hpp index f459ed4b6c..1298bb36d4 100644 --- a/modules/core/include/opencv2/core/mat.hpp +++ b/modules/core/include/opencv2/core/mat.hpp @@ -1305,15 +1305,36 @@ public: t(); // finally, transpose the Nx3 matrix. // This involves copying all the elements @endcode + 3-channel 2x2 matrix reshaped to 1-channel 4x3 matrix, each column has values from one of original channels: + @code + Mat m(Size(2, 2), CV_8UC3, Scalar(1, 2, 3)); + vector new_shape {4, 3}; + m = m.reshape(1, new_shape); + @endcode + or: + @code + Mat m(Size(2, 2), CV_8UC3, Scalar(1, 2, 3)); + const int new_shape[] = {4, 3}; + m = m.reshape(1, 2, new_shape); + @endcode @param cn New number of channels. If the parameter is 0, the number of channels remains the same. @param rows New number of rows. If the parameter is 0, the number of rows remains the same. */ Mat reshape(int cn, int rows=0) const; - /** @overload */ + /** @overload + * @param cn New number of channels. If the parameter is 0, the number of channels remains the same. + * @param newndims New number of dimentions. + * @param newsz Array with new matrix size by all dimentions. If some sizes are zero, + * the original sizes in those dimensions are presumed. + */ Mat reshape(int cn, int newndims, const int* newsz) const; - /** @overload */ + /** @overload + * @param cn New number of channels. If the parameter is 0, the number of channels remains the same. + * @param newshape Vector with new matrix size by all dimentions. If some sizes are zero, + * the original sizes in those dimensions are presumed. + */ Mat reshape(int cn, const std::vector& newshape) const; /** @brief Transposes a matrix. From fe93724d3feed581d34adc2ee342fd7b835a0a3a Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 21 Jun 2023 09:22:26 +0300 Subject: [PATCH 028/105] FFmpeg wrapper update to FFmpeg version 3.4.13. --- 3rdparty/ffmpeg/ffmpeg.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index ad10940335..705766e71c 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -1,8 +1,8 @@ -# Binaries branch name: ffmpeg/3.4_20211220 -# Binaries were created for OpenCV: a22dd28e0272ec0f1cfee8811d3f5f0392827c65 -ocv_update(FFMPEG_BINARIES_COMMIT "5a7644ec3940c6eed41c6ebb5a0602a5615fdb3f") -ocv_update(FFMPEG_FILE_HASH_BIN32 "8ad9de6f1f2ca77786748d1f3a4e83ea") -ocv_update(FFMPEG_FILE_HASH_BIN64 "2c670f068252e7cd28d3883993dc1d6e") +# Binaries branch name: 3.4_20230620 +# Binaries were created for OpenCV: c97c22b7cf2ef0f82cd4203a2e9a6eda94e9f7f1 +ocv_update(FFMPEG_BINARIES_COMMIT "7c4bb90fd43a13732ae907981a88fb983a7e2197") +ocv_update(FFMPEG_FILE_HASH_BIN32 "d7db86de29b0460294489c5ed3180b56") +ocv_update(FFMPEG_FILE_HASH_BIN64 "9df93d8afff2eee368ad484098a12b18") ocv_update(FFMPEG_FILE_HASH_CMAKE "3b90f67f4b429e77d3da36698cef700c") function(download_win_ffmpeg script_var) From 1db1422fbdc1a03d1523b50ad9b05d72cbd39616 Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Wed, 21 Jun 2023 18:11:39 +0800 Subject: [PATCH 029/105] Merge pull request #23829 from fengyuentau:fixes4orbbec Fix broken links and outdated information for documentation of Orbbec camera #23829 Resolves the documentation issue from https://github.com/opencv/opencv/issues/23579. Orbbec is moving to support UVC directly so they do not provide the old `install.sh` for OpenNI SDK >= 2.3.0.86. Also in their new release of OpenNI SDK, paths of include headers and libraries are changed. Changing our cmake script for this change does not make sense since we cannot make this kind of change everytime they update. So just added a subsection providing `install.sh` for users as a workaround on our side. @Lecrapouille You may also take a look at this pull request. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- doc/tutorials/app/orbbec_astra.markdown | 45 ++++++++++++++++++++----- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/doc/tutorials/app/orbbec_astra.markdown b/doc/tutorials/app/orbbec_astra.markdown index b95053b8ea..01a685db57 100644 --- a/doc/tutorials/app/orbbec_astra.markdown +++ b/doc/tutorials/app/orbbec_astra.markdown @@ -9,7 +9,7 @@ Using Orbbec Astra 3D cameras {#tutorial_orbbec_astra} ### Introduction -This tutorial is devoted to the Astra Series of Orbbec 3D cameras (https://orbbec3d.com/product-astra-pro/). +This tutorial is devoted to the Astra Series of Orbbec 3D cameras (https://orbbec3d.com/index/Product/info.html?cate=38&id=36). That cameras have a depth sensor in addition to a common color sensor. The depth sensors can be read using the open source OpenNI API with @ref cv::VideoCapture class. The video stream is provided through the regular camera interface. @@ -18,9 +18,11 @@ camera interface. In order to use the Astra camera's depth sensor with OpenCV you should do the following steps: --# Download the latest version of Orbbec OpenNI SDK (from here ). +-# Download the latest version of Orbbec OpenNI SDK (from here ). Unzip the archive, choose the build according to your operating system and follow installation - steps provided in the Readme file. For instance, if you use 64bit GNU/Linux run: + steps provided in the Readme file. + +-# For instance, if you use 64bit GNU/Linux run: @code{.bash} $ cd Linux/OpenNI-Linux-x64-2.3.0.63/ $ sudo ./install.sh @@ -31,17 +33,44 @@ In order to use the Astra camera's depth sensor with OpenCV you should do the fo @code{.bash} $ source OpenNIDevEnvironment @endcode - --# Run the following commands to verify that OpenNI library and header files can be found. You should see - something similar in your terminal: + To verify that the source command works and OpenNI library and header files can be found, run the following + command and you should see something similar in your terminal: @code{.bash} $ echo $OPENNI2_INCLUDE /home/user/OpenNI_2.3.0.63/Linux/OpenNI-Linux-x64-2.3.0.63/Include $ echo $OPENNI2_REDIST /home/user/OpenNI_2.3.0.63/Linux/OpenNI-Linux-x64-2.3.0.63/Redist @endcode - If the above two variables are empty, then you need to source `OpenNIDevEnvironment` again. Now you can - configure OpenCV with OpenNI support enabled by setting the `WITH_OPENNI2` flag in CMake. + If the above two variables are empty, then you need to source `OpenNIDevEnvironment` again. + + @note Orbbec OpenNI SDK version 2.3.0.86 and newer does not provide `install.sh` any more. + You can use the following script to initialize environment: + @code{.text} + # Check if user is root/running with sudo + if [ `whoami` != root ]; then + echo Please run this script with sudo + exit + fi + + ORIG_PATH=`pwd` + cd `dirname $0` + SCRIPT_PATH=`pwd` + cd $ORIG_PATH + + if [ "`uname -s`" != "Darwin" ]; then + # Install UDEV rules for USB device + cp ${SCRIPT_PATH}/orbbec-usb.rules /etc/udev/rules.d/558-orbbec-usb.rules + echo "usb rules file install at /etc/udev/rules.d/558-orbbec-usb.rules" + fi + + OUT_FILE="$SCRIPT_PATH/OpenNIDevEnvironment" + echo "export OPENNI2_INCLUDE=$SCRIPT_PATH/../sdk/Include" > $OUT_FILE + echo "export OPENNI2_REDIST=$SCRIPT_PATH/../sdk/libs" >> $OUT_FILE + chmod a+r $OUT_FILE + echo "exit" + @endcode + +-# Now you can configure OpenCV with OpenNI support enabled by setting the `WITH_OPENNI2` flag in CMake. You may also like to enable the `BUILD_EXAMPLES` flag to get a code sample working with your Astra camera. Run the following commands in the directory containing OpenCV source code to enable OpenNI support: @code{.bash} From 1656e7573e043e28e2c73697ce0b6466aa0c7cc3 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Wed, 21 Jun 2023 14:14:53 +0000 Subject: [PATCH 030/105] gapi: fix static build with openvino --- modules/gapi/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index b9f2c11a2f..31322d533a 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -254,6 +254,7 @@ ocv_target_link_libraries(${the_module} PRIVATE ade) if(TARGET ocv.3rdparty.openvino AND OPENCV_GAPI_WITH_OPENVINO) ocv_target_link_libraries(${the_module} PRIVATE ocv.3rdparty.openvino) + ocv_install_used_external_targets(ocv.3rdparty.openvino) endif() if(HAVE_TBB) From 60848519b5352cf06f2fdef1f140af1179cfc903 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Thu, 22 Jun 2023 10:46:25 +0100 Subject: [PATCH 031/105] Merge pull request #23843 from TolyaTalamanov:at/fix-missing-opaque-kind-for-kernel G-API: Fix incorrect OpaqueKind for Kernel outputs #23843 ### Pull Request Readiness Checklist #### Overview The PR is going to fix several problems: 1. Major: `GKernel` doesn't hold `kind` for its outputs. Since `GModelBuilder` traverse graph from outputs to inputs once it reaches any output of the operation it will use its `kind` to create `Data` meta for all operation outputs. Since it essential for `python` to know `GTypeInfo` (which is `shape` and `kind`) it will be confused. Consider this operation: ``` @cv.gapi.op('custom.square_mean', in_types=[cv.GArray.Int], out_types=[cv.GOpaque.Float, cv.GArray.Int]) class GSquareMean: @staticmethod def outMeta(desc): return cv.empty_gopaque_desc(), cv.empty_array_desc() ``` Even though `GOpaque` is `Float`, corresponding metadata might have `Int` kind because it might be taken from `cv.GArray.Int` so it will be a problem if one of the outputs of these operation is graph output because python will cast it to the wrong type based on `Data` meta. 2. Minor: Some of the OpenVINO `IR`'s doesn't any layout information for input. It's usually true only for `IRv10` but since `OpenVINO 2.0` need this information to correctly configure resize we need to put default layout if there no such assigned in `ov::Model`. See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/gapi/include/opencv2/gapi/gkernel.hpp | 7 +++- modules/gapi/include/opencv2/gapi/infer.hpp | 3 ++ .../include/opencv2/gapi/streaming/desync.hpp | 1 + .../include/opencv2/gapi/streaming/meta.hpp | 1 + modules/gapi/misc/python/python_bridge.hpp | 27 ++++++++---- .../python/test/test_gapi_sample_pipelines.py | 41 +++++++++++++++++++ modules/gapi/src/backends/ov/govbackend.cpp | 14 ++++++- modules/gapi/src/compiler/gmodelbuilder.cpp | 18 ++++---- .../internal/gapi_int_gmodel_builder_test.cpp | 6 ++- 9 files changed, 95 insertions(+), 23 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/gkernel.hpp b/modules/gapi/include/opencv2/gapi/gkernel.hpp index 50d2efdd55..1b910adc82 100644 --- a/modules/gapi/include/opencv2/gapi/gkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/gkernel.hpp @@ -51,6 +51,7 @@ struct GAPI_EXPORTS GKernel GShapes outShapes; // types (shapes) kernel's outputs GKinds inKinds; // kinds of kernel's inputs (fixme: below) GCtors outCtors; // captured constructors for template output types + GKinds outKinds; // kinds of kernel's outputs (fixme: below) }; // TODO: It's questionable if inKinds should really be here. Instead, // this information could come from meta. @@ -227,7 +228,8 @@ public: , &K::getOutMeta , {detail::GTypeTraits::shape...} , {detail::GTypeTraits::op_kind...} - , {detail::GObtainCtor::get()...}}); + , {detail::GObtainCtor::get()...} + , {detail::GTypeTraits::op_kind...}}); call.pass(args...); // TODO: std::forward() here? return yield(call, typename detail::MkSeq::type()); } @@ -251,7 +253,8 @@ public: , &K::getOutMeta , {detail::GTypeTraits::shape} , {detail::GTypeTraits::op_kind...} - , {detail::GObtainCtor::get()}}); + , {detail::GObtainCtor::get()} + , {detail::GTypeTraits::op_kind}}); call.pass(args...); return detail::Yield::yield(call, 0); } diff --git a/modules/gapi/include/opencv2/gapi/infer.hpp b/modules/gapi/include/opencv2/gapi/infer.hpp index c1f9501873..abbd32ba20 100644 --- a/modules/gapi/include/opencv2/gapi/infer.hpp +++ b/modules/gapi/include/opencv2/gapi/infer.hpp @@ -101,8 +101,10 @@ public: if (it == m_priv->blobs.end()) { // FIXME: Avoid modifying GKernel auto shape = cv::detail::GTypeTraits::shape; + auto kind = cv::detail::GTypeTraits::op_kind; m_priv->call->kernel().outShapes.push_back(shape); m_priv->call->kernel().outCtors.emplace_back(cv::detail::GObtainCtor::get()); + m_priv->call->kernel().outKinds.emplace_back(kind); auto out_idx = static_cast(m_priv->blobs.size()); it = m_priv->blobs.emplace(name, cv::detail::Yield::yield(*(m_priv->call), out_idx)).first; @@ -175,6 +177,7 @@ std::shared_ptr makeCall(const std::string &tag, {}, // outShape will be filled later std::move(kinds), {}, // outCtors will be filled later + {}, // outKinds will be filled later }); call->setArgs(std::move(args)); diff --git a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp index 9e927872a3..0e04f5beb9 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/desync.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/desync.hpp @@ -46,6 +46,7 @@ G desync(const G &g) { , {cv::detail::GTypeTraits::shape} // output Shape , {cv::detail::GTypeTraits::op_kind} // input data kinds , {cv::detail::GObtainCtor::get()} // output template ctors + , {cv::detail::GTypeTraits::op_kind} // output data kinds }; cv::GCall call(std::move(k)); call.pass(g); diff --git a/modules/gapi/include/opencv2/gapi/streaming/meta.hpp b/modules/gapi/include/opencv2/gapi/streaming/meta.hpp index cbcfc3aa37..cdd3d371cb 100644 --- a/modules/gapi/include/opencv2/gapi/streaming/meta.hpp +++ b/modules/gapi/include/opencv2/gapi/streaming/meta.hpp @@ -50,6 +50,7 @@ cv::GOpaque meta(G g, const std::string &tag) { , {cv::detail::GTypeTraits::shape} // output Shape , {cv::detail::GTypeTraits::op_kind} // input data kinds , {cv::detail::GObtainCtor::get()} // output template ctors + , {cv::detail::GTypeTraits::op_kind} // output data kind }; cv::GCall call(std::move(k)); call.pass(g); diff --git a/modules/gapi/misc/python/python_bridge.hpp b/modules/gapi/misc/python/python_bridge.hpp index 07d056abb7..53edf38b30 100644 --- a/modules/gapi/misc/python/python_bridge.hpp +++ b/modules/gapi/misc/python/python_bridge.hpp @@ -267,13 +267,14 @@ cv::gapi::wip::GOutputs::Priv::Priv(const std::string& id, cv::GKernel::M outMet std::transform(args.begin(), args.end(), std::back_inserter(kinds), [](const cv::GArg& arg) { return arg.opaque_kind; }); - m_call.reset(new cv::GCall{cv::GKernel{id, {}, outMeta, {}, std::move(kinds), {}}}); + m_call.reset(new cv::GCall{cv::GKernel{id, {}, outMeta, {}, std::move(kinds), {}, {}}}); m_call->setArgs(std::move(args)); } cv::GMat cv::gapi::wip::GOutputs::Priv::getGMat() { m_call->kernel().outShapes.push_back(cv::GShape::GMAT); + m_call->kernel().outKinds.push_back(cv::detail::OpaqueKind::CV_UNKNOWN); // ...so _empty_ constructor is passed here. m_call->kernel().outCtors.emplace_back(cv::util::monostate{}); return m_call->yield(output++); @@ -282,6 +283,7 @@ cv::GMat cv::gapi::wip::GOutputs::Priv::getGMat() cv::GScalar cv::gapi::wip::GOutputs::Priv::getGScalar() { m_call->kernel().outShapes.push_back(cv::GShape::GSCALAR); + m_call->kernel().outKinds.push_back(cv::detail::OpaqueKind::CV_UNKNOWN); // ...so _empty_ constructor is passed here. m_call->kernel().outCtors.emplace_back(cv::util::monostate{}); return m_call->yieldScalar(output++); @@ -290,10 +292,14 @@ cv::GScalar cv::gapi::wip::GOutputs::Priv::getGScalar() cv::GArrayT cv::gapi::wip::GOutputs::Priv::getGArray(cv::gapi::ArgType type) { m_call->kernel().outShapes.push_back(cv::GShape::GARRAY); -#define HC(T, K) \ - case K: \ - m_call->kernel().outCtors.emplace_back(cv::detail::GObtainCtor>::get()); \ - return cv::GArrayT(m_call->yieldArray(output++)); \ + +#define HC(T, K) \ + case K: { \ + const auto kind = cv::detail::GTypeTraits>::op_kind; \ + m_call->kernel().outKinds.emplace_back(kind); \ + m_call->kernel().outCtors.emplace_back(cv::detail::GObtainCtor>::get()); \ + return cv::GArrayT(m_call->yieldArray(output++)); \ + } SWITCH(type, GARRAY_TYPE_LIST_G, HC) #undef HC @@ -302,10 +308,13 @@ cv::GArrayT cv::gapi::wip::GOutputs::Priv::getGArray(cv::gapi::ArgType type) cv::GOpaqueT cv::gapi::wip::GOutputs::Priv::getGOpaque(cv::gapi::ArgType type) { m_call->kernel().outShapes.push_back(cv::GShape::GOPAQUE); -#define HC(T, K) \ - case K: \ - m_call->kernel().outCtors.emplace_back(cv::detail::GObtainCtor>::get()); \ - return cv::GOpaqueT(m_call->yieldOpaque(output++)); \ +#define HC(T, K) \ + case K: { \ + const auto kind = cv::detail::GTypeTraits>::op_kind; \ + m_call->kernel().outKinds.emplace_back(kind); \ + m_call->kernel().outCtors.emplace_back(cv::detail::GObtainCtor>::get()); \ + return cv::GOpaqueT(m_call->yieldOpaque(output++)); \ + } SWITCH(type, GOPAQUE_TYPE_LIST_G, HC) #undef HC diff --git a/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py b/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py index 8e15d457d9..fbe9aafa54 100644 --- a/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py +++ b/modules/gapi/misc/python/test/test_gapi_sample_pipelines.py @@ -207,7 +207,48 @@ try: return Op + # NB: Just mock operation to test different kinds for output G-types. + @cv.gapi.op('custom.square_mean', in_types=[cv.GArray.Int], out_types=[cv.GOpaque.Float, cv.GArray.Int]) + class GSquareMean: + @staticmethod + def outMeta(desc): + return cv.empty_gopaque_desc(), cv.empty_array_desc() + + + @cv.gapi.kernel(GSquareMean) + class GSquareMeanImpl: + @staticmethod + def run(arr): + squares = [val**2 for val in arr] + return sum(arr) / len(arr), squares + + @cv.gapi.op('custom.squares', in_types=[cv.GArray.Int], out_types=[cv.GArray.Int]) + class GSquare: + @staticmethod + def outMeta(desc): + return cv.empty_array_desc() + + + @cv.gapi.kernel(GSquare) + class GSquareImpl: + @staticmethod + def run(arr): + squares = [val**2 for val in arr] + return squares + + class gapi_sample_pipelines(NewOpenCVTests): + def test_different_output_opaque_kinds(self): + g_in = cv.GArray.Int() + g_mean, g_squares = GSquareMean.on(g_in) + comp = cv.GComputation(cv.GIn(g_in), cv.GOut(g_mean, g_squares)) + + pkg = cv.gapi.kernels(GSquareMeanImpl) + mean, squares = comp.apply(cv.gin([1,2,3]), args=cv.gapi.compile_args(pkg)) + + self.assertEqual([1,4,9], list(squares)) + self.assertEqual(2.0, mean) + def test_custom_op_add(self): sz = (3, 3) diff --git a/modules/gapi/src/backends/ov/govbackend.cpp b/modules/gapi/src/backends/ov/govbackend.cpp index 0fff90bb4c..9497338021 100644 --- a/modules/gapi/src/backends/ov/govbackend.cpp +++ b/modules/gapi/src/backends/ov/govbackend.cpp @@ -738,6 +738,15 @@ public: const auto explicit_in_model_layout = lookUp(m_input_model_layout, input_name); if (explicit_in_model_layout) { input_info.model().set_layout(::ov::Layout(*explicit_in_model_layout)); + } else if (m_model->input(input_name).get_shape().size() == 4u) { + // NB: Back compatibility with IR's without any layout information. + // Note that default is only applicable for 4D inputs in order to + // support auto resize for image use cases. + GAPI_LOG_WARNING(NULL, "Failed to find layout for input layer \"" + << input_name << "\" - NCHW is set by default"); + const std::string default_layout = "NCHW"; + input_info.model().set_layout(::ov::Layout(default_layout)); + m_input_model_layout.emplace(input_name, default_layout); } const auto explicit_in_tensor_layout = lookUp(m_input_tensor_layout, input_name); if (explicit_in_tensor_layout) { @@ -765,6 +774,7 @@ public: const auto &matdesc = cv::util::get(input_meta); const auto explicit_in_tensor_layout = lookUp(m_input_tensor_layout, input_name); + const auto explicit_in_model_layout = lookUp(m_input_model_layout, input_name); const auto explicit_resize = lookUp(m_interpolation, input_name); if (disable_img_resize && explicit_resize.has_value()) { @@ -810,7 +820,9 @@ public: if (matdesc.isND()) { // NB: ND case - need to obtain "H" and "W" positions // in order to configure resize. - const auto model_layout = ::ov::layout::get_layout(m_model->input(input_name)); + const auto model_layout = explicit_in_model_layout + ? ::ov::Layout(*explicit_in_model_layout) + : ::ov::layout::get_layout(m_model->input(input_name)); if (!explicit_in_tensor_layout && model_layout.empty()) { std::stringstream ss; ss << "Resize for input layer: " << input_name diff --git a/modules/gapi/src/compiler/gmodelbuilder.cpp b/modules/gapi/src/compiler/gmodelbuilder.cpp index aed3428693..eb807e74cd 100644 --- a/modules/gapi/src/compiler/gmodelbuilder.cpp +++ b/modules/gapi/src/compiler/gmodelbuilder.cpp @@ -59,7 +59,6 @@ private: } // namespace - cv::gimpl::Unrolled cv::gimpl::unrollExpr(const GProtoArgs &ins, const GProtoArgs &outs) { @@ -135,18 +134,19 @@ cv::gimpl::Unrolled cv::gimpl::unrollExpr(const GProtoArgs &ins, // Put the outputs object description of the node // so that they are not lost if they are not consumed by other operations GAPI_Assert(call_p.m_k.outCtors.size() == call_p.m_k.outShapes.size()); - for (const auto it : ade::util::indexed(call_p.m_k.outShapes)) + for (const auto it : ade::util::indexed(ade::util::zip(call_p.m_k.outShapes, + call_p.m_k.outCtors, + call_p.m_k.outKinds))) { - std::size_t port = ade::util::index(it); - GShape shape = ade::util::value(it); - - // FIXME: then use ZIP - HostCtor ctor = call_p.m_k.outCtors[port]; - + auto port = ade::util::index(it); + auto &val = ade::util::value(it); + auto shape = std::get<0>(val); + auto ctor = std::get<1>(val); + auto kind = std::get<2>(val); // NB: Probably this fixes all other "missing host ctor" // problems. // TODO: Clean-up the old workarounds if it really is. - GOrigin org {shape, node, port, std::move(ctor), origin.kind}; + GOrigin org {shape, node, port, std::move(ctor), kind}; origins.insert(org); } diff --git a/modules/gapi/test/internal/gapi_int_gmodel_builder_test.cpp b/modules/gapi/test/internal/gapi_int_gmodel_builder_test.cpp index 4291fd00e9..74de7aaf59 100644 --- a/modules/gapi/test/internal/gapi_int_gmodel_builder_test.cpp +++ b/modules/gapi/test/internal/gapi_int_gmodel_builder_test.cpp @@ -30,7 +30,8 @@ namespace , nullptr , { GShape::GMAT } , { D::OpaqueKind::CV_UNKNOWN } - , { cv::detail::HostCtor{cv::util::monostate{}} } + , { D::HostCtor{cv::util::monostate{}} } + , { D::OpaqueKind::CV_UNKNOWN } }).pass(m).yield(0); } @@ -41,7 +42,8 @@ namespace , nullptr , { GShape::GMAT } , { D::OpaqueKind::CV_UNKNOWN, D::OpaqueKind::CV_UNKNOWN } - , { cv::detail::HostCtor{cv::util::monostate{}} } + , { D::HostCtor{cv::util::monostate{}} } + , { D::OpaqueKind::CV_UNKNOWN} }).pass(m1, m2).yield(0); } From 426b0887549724eb9d621ab841e31c60fb9940a5 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Thu, 22 Jun 2023 13:13:10 +0300 Subject: [PATCH 032/105] FFmpeg/4.x: update FFmpeg wrapper 2023.6 --- 3rdparty/ffmpeg/ffmpeg.cmake | 10 +++++----- modules/videoio/test/test_ffmpeg.cpp | 9 --------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/3rdparty/ffmpeg/ffmpeg.cmake b/3rdparty/ffmpeg/ffmpeg.cmake index f3ad63770a..da75e3d2ca 100644 --- a/3rdparty/ffmpeg/ffmpeg.cmake +++ b/3rdparty/ffmpeg/ffmpeg.cmake @@ -1,8 +1,8 @@ -# Binaries branch name: ffmpeg/4.x_20221225 -# Binaries were created for OpenCV: 4abe6dc48d4ec6229f332cc6cf6c7e234ac8027e -ocv_update(FFMPEG_BINARIES_COMMIT "7dd0d4f1d6fe75f05f3d3b5e38cbc96c1a2d2809") -ocv_update(FFMPEG_FILE_HASH_BIN32 "e598ae2ece1ddf310bc49b58202fd87a") -ocv_update(FFMPEG_FILE_HASH_BIN64 "b2a40c142c20aef9fd663fc8f85c2971") +# Binaries branch name: ffmpeg/4.x_20230622 +# Binaries were created for OpenCV: 61d48dd0f8d1cc1a115d26998705a61478f64a3c +ocv_update(FFMPEG_BINARIES_COMMIT "7da61f0695eabf8972a2c302bf1632a3d99fb0d5") +ocv_update(FFMPEG_FILE_HASH_BIN32 "4aaef1456e282e5ef665d65555f47f56") +ocv_update(FFMPEG_FILE_HASH_BIN64 "38a638851e064c591ce812e27ed43f1f") ocv_update(FFMPEG_FILE_HASH_CMAKE "8862c87496e2e8c375965e1277dee1c7") function(download_win_ffmpeg script_var) diff --git a/modules/videoio/test/test_ffmpeg.cpp b/modules/videoio/test/test_ffmpeg.cpp index e0c2a44075..35d425d5c1 100644 --- a/modules/videoio/test/test_ffmpeg.cpp +++ b/modules/videoio/test/test_ffmpeg.cpp @@ -226,11 +226,7 @@ const videoio_container_params_t videoio_container_params[] = videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "h264", "h264", "h264", "I420"), videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "h265", "h265", "hevc", "I420"), videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "mjpg.avi", "mjpg", "MJPG", "I420"), -#ifdef _WIN32 // handle old FFmpeg backend - remove when windows shared library is updated - videoio_container_params_t(CAP_FFMPEG, "video/sample_322x242_15frames.yuv420p.libx264", "mp4", "h264", "avc1", "I420") -#else videoio_container_params_t(CAP_FFMPEG, "video/sample_322x242_15frames.yuv420p.libx264", "mp4", "h264", "h264", "I420") -#endif //videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "h264.mkv", "mkv.h264", "h264", "I420"), //videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "h265.mkv", "mkv.h265", "hevc", "I420"), //videoio_container_params_t(CAP_FFMPEG, "video/big_buck_bunny", "h264.mp4", "mp4.avc1", "avc1", "I420"), @@ -454,10 +450,6 @@ const ffmpeg_get_fourcc_param_t ffmpeg_get_fourcc_param[] = ffmpeg_get_fourcc_param_t("../cv/tracking/faceocc2/data/faceocc2.webm", "VP80"), ffmpeg_get_fourcc_param_t("video/big_buck_bunny.h265", "hevc"), ffmpeg_get_fourcc_param_t("video/big_buck_bunny.h264", "h264"), -#ifdef _WIN32 // handle old FFmpeg backend - remove when windows shared library is updated - ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", "vp09"), - ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libaom-av1.mp4", "av01") -#else ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", "VP90"), ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libaom-av1.mp4", "AV01"), ffmpeg_get_fourcc_param_t("video/big_buck_bunny.mp4", "FMP4"), @@ -466,7 +458,6 @@ const ffmpeg_get_fourcc_param_t ffmpeg_get_fourcc_param[] = ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libxvid.mp4", "FMP4"), ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libx265.mp4", "hevc"), ffmpeg_get_fourcc_param_t("video/sample_322x242_15frames.yuv420p.libx264.mp4", "h264") -#endif }; INSTANTIATE_TEST_CASE_P(videoio, ffmpeg_get_fourcc, testing::ValuesIn(ffmpeg_get_fourcc_param)); From 22b747eae26e6724b1fd8ee82a004ced7425614a Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Thu, 22 Jun 2023 15:09:53 +0300 Subject: [PATCH 033/105] Merge pull request #23702 from dkurt:py_rotated_rect Python binding for RotatedRect #23702 ### Pull Request Readiness Checklist related: https://github.com/opencv/opencv/issues/23546#issuecomment-1562894602 See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/core/include/opencv2/core/types.hpp | 19 ++++++++------- modules/core/src/types.cpp | 5 ++++ modules/python/src2/cv2_convert.cpp | 26 +++++++++++++++++++++ modules/python/src2/pycompat.hpp | 10 ++++++-- modules/python/test/test_misc.py | 21 +++++++++++++++++ 5 files changed, 71 insertions(+), 10 deletions(-) diff --git a/modules/core/include/opencv2/core/types.hpp b/modules/core/include/opencv2/core/types.hpp index 73cb29f73a..844d8f8950 100644 --- a/modules/core/include/opencv2/core/types.hpp +++ b/modules/core/include/opencv2/core/types.hpp @@ -527,23 +527,23 @@ The sample below demonstrates how to use RotatedRect: @sa CamShift, fitEllipse, minAreaRect, CvBox2D */ -class CV_EXPORTS RotatedRect +class CV_EXPORTS_W_SIMPLE RotatedRect { public: //! default constructor - RotatedRect(); + CV_WRAP RotatedRect(); /** full constructor @param center The rectangle mass center. @param size Width and height of the rectangle. @param angle The rotation angle in a clockwise direction. When the angle is 0, 90, 180, 270 etc., the rectangle becomes an up-right rectangle. */ - RotatedRect(const Point2f& center, const Size2f& size, float angle); + CV_WRAP RotatedRect(const Point2f& center, const Size2f& size, float angle); /** Any 3 end points of the RotatedRect. They must be given in order (either clockwise or anticlockwise). */ - RotatedRect(const Point2f& point1, const Point2f& point2, const Point2f& point3); + CV_WRAP RotatedRect(const Point2f& point1, const Point2f& point2, const Point2f& point3); /** returns 4 vertices of the rotated rectangle @param pts The points array for storing rectangle vertices. The order is _bottomLeft_, _topLeft_, topRight, bottomRight. @@ -552,16 +552,19 @@ public: rectangle. */ void points(Point2f pts[]) const; + + CV_WRAP void points(CV_OUT std::vector& pts) const; + //! returns the minimal up-right integer rectangle containing the rotated rectangle - Rect boundingRect() const; + CV_WRAP Rect boundingRect() const; //! returns the minimal (exact) floating point rectangle containing the rotated rectangle, not intended for use with images Rect_ boundingRect2f() const; //! returns the rectangle mass center - Point2f center; + CV_PROP_RW Point2f center; //! returns width and height of the rectangle - Size2f size; + CV_PROP_RW Size2f size; //! returns the rotation angle. When the angle is 0, 90, 180, 270 etc., the rectangle becomes an up-right rectangle. - float angle; + CV_PROP_RW float angle; }; template<> class DataType< RotatedRect > diff --git a/modules/core/src/types.cpp b/modules/core/src/types.cpp index 15ba83a0f6..43e25ef045 100644 --- a/modules/core/src/types.cpp +++ b/modules/core/src/types.cpp @@ -186,6 +186,11 @@ void RotatedRect::points(Point2f pt[]) const pt[3].y = 2*center.y - pt[1].y; } +void RotatedRect::points(std::vector& pts) const { + pts.resize(4); + points(pts.data()); +} + Rect RotatedRect::boundingRect() const { Point2f pt[4]; diff --git a/modules/python/src2/cv2_convert.cpp b/modules/python/src2/cv2_convert.cpp index f03a2e2d86..2e69586f47 100644 --- a/modules/python/src2/cv2_convert.cpp +++ b/modules/python/src2/cv2_convert.cpp @@ -728,6 +728,26 @@ PyObject* pyopencv_from(const Rect2d& r) // --- RotatedRect +static inline bool convertToRotatedRect(PyObject* obj, RotatedRect& dst) +{ + PyObject* type = PyObject_Type(obj); + if (getPyObjectAttr(type, "__module__") == MODULESTR && + getPyObjectNameAttr(type) == "RotatedRect") + { + struct pyopencv_RotatedRect_t + { + PyObject_HEAD + cv::RotatedRect v; + }; + dst = reinterpret_cast(obj)->v; + + Py_DECREF(type); + return true; + } + Py_DECREF(type); + return false; +} + template<> bool pyopencv_to(PyObject* obj, RotatedRect& dst, const ArgInfo& info) { @@ -735,6 +755,12 @@ bool pyopencv_to(PyObject* obj, RotatedRect& dst, const ArgInfo& info) { return true; } + // This is a workaround for compatibility with an initialization from tuple. + // Allows import RotatedRect as an object. + if (convertToRotatedRect(obj, dst)) + { + return true; + } if (!PySequence_Check(obj)) { failmsg("Can't parse '%s' as RotatedRect." diff --git a/modules/python/src2/pycompat.hpp b/modules/python/src2/pycompat.hpp index bd3956dbc0..c8806dc812 100644 --- a/modules/python/src2/pycompat.hpp +++ b/modules/python/src2/pycompat.hpp @@ -98,10 +98,10 @@ static inline bool getUnicodeString(PyObject * obj, std::string &str) } static inline -std::string getPyObjectNameAttr(PyObject* obj) +std::string getPyObjectAttr(PyObject* obj, const char* attrName) { std::string obj_name; - PyObject* cls_name_obj = PyObject_GetAttrString(obj, "__name__"); + PyObject* cls_name_obj = PyObject_GetAttrString(obj, attrName); if (cls_name_obj && !getUnicodeString(cls_name_obj, obj_name)) { obj_name.clear(); } @@ -117,6 +117,12 @@ std::string getPyObjectNameAttr(PyObject* obj) return obj_name; } +static inline +std::string getPyObjectNameAttr(PyObject* obj) +{ + return getPyObjectAttr(obj, "__name__"); +} + //================================================================================================== #define CV_PY_FN_WITH_KW_(fn, flags) (PyCFunction)(void*)(PyCFunctionWithKeywords)(fn), (flags) | METH_VARARGS | METH_KEYWORDS diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index e8c4ab8849..ec7f44de0f 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -482,6 +482,27 @@ class Arguments(NewOpenCVTests): self.assertEqual(expected, actual, msg=get_conversion_error_msg(convertible, expected, actual)) + + def test_wrap_rotated_rect(self): + center = (34.5, 52.) + size = (565.0, 140.0) + angle = -177.5 + rect1 = cv.RotatedRect(center, size, angle) + self.assertEqual(rect1.center, center) + self.assertEqual(rect1.size, size) + self.assertEqual(rect1.angle, angle) + + pts = [[ 319.7845, -5.6109037], + [ 313.6778, 134.25586], + [-250.78448, 109.6109], + [-244.6778, -30.25586]] + self.assertLess(np.max(np.abs(rect1.points() - pts)), 1e-4) + + rect2 = cv.RotatedRect(pts[0], pts[1], pts[2]) + _, inter_pts = cv.rotatedRectangleIntersection(rect1, rect2) + self.assertLess(np.max(np.abs(inter_pts.reshape(-1, 2) - pts)), 1e-4) + + def test_parse_to_rotated_rect_not_convertible(self): for not_convertible in ([], (), np.array([]), (123, (45, 34), 1), {1: 2, 3: 4}, 123, np.array([[123, 123, 14], [1, 3], 56], dtype=object), '123'): From affc69bf1fc070c07739bd5b68c14d82c67175b1 Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Thu, 22 Jun 2023 17:30:44 +0300 Subject: [PATCH 034/105] Merge pull request #23848 from AleksandrPanov:fix_detectDiamonds_api Fix detect diamonds api #23848 `detectDiamonds` cannot be called from python, reproducer: ``` import numpy as np import cv2 as cv detector = cv.aruco.CharucoDetector( cv.aruco.CharucoBoard( (3, 3), 200.0, 100.0, cv.aruco.getPredefinedDictionary(cv.aruco.DICT_4X4_250) ) ) image = np.zeros((640, 480, 1), dtype=np.uint8) res = detector.detectDiamonds(image) print(res) ``` The error in `detectDiamonds` API fixed by replacing `InputOutputArrayOfArrays markerIds` with `InputOutputArray markerIds`. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- .../opencv2/objdetect/charuco_detector.hpp | 2 +- .../misc/python/test/test_objdetect_aruco.py | 21 +++++++++++++++++++ .../objdetect/src/aruco/charuco_detector.cpp | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/modules/objdetect/include/opencv2/objdetect/charuco_detector.hpp b/modules/objdetect/include/opencv2/objdetect/charuco_detector.hpp index 15170b4d2e..a23960d557 100644 --- a/modules/objdetect/include/opencv2/objdetect/charuco_detector.hpp +++ b/modules/objdetect/include/opencv2/objdetect/charuco_detector.hpp @@ -104,7 +104,7 @@ public: */ CV_WRAP void detectDiamonds(InputArray image, OutputArrayOfArrays diamondCorners, OutputArray diamondIds, InputOutputArrayOfArrays markerCorners = noArray(), - InputOutputArrayOfArrays markerIds = noArray()) const; + InputOutputArray markerIds = noArray()) const; protected: struct CharucoDetectorImpl; Ptr charucoDetectorImpl; diff --git a/modules/objdetect/misc/python/test/test_objdetect_aruco.py b/modules/objdetect/misc/python/test/test_objdetect_aruco.py index eb88f2c8f6..d63a19cd2f 100644 --- a/modules/objdetect/misc/python/test/test_objdetect_aruco.py +++ b/modules/objdetect/misc/python/test/test_objdetect_aruco.py @@ -238,6 +238,27 @@ class aruco_objdetect_test(NewOpenCVTests): self.assertEqual(charucoIds[i], i) np.testing.assert_allclose(gold_corners, charucoCorners.reshape(-1, 2), 0.01, 0.1) + def test_detect_diamonds(self): + aruco_dict = cv.aruco.getPredefinedDictionary(cv.aruco.DICT_6X6_250) + board_size = (3, 3) + board = cv.aruco.CharucoBoard(board_size, 1.0, .8, aruco_dict) + charuco_detector = cv.aruco.CharucoDetector(board) + cell_size = 120 + + image = board.generateImage((cell_size*board_size[0], cell_size*board_size[1])) + + list_gold_corners = [(cell_size, cell_size), (2*cell_size, cell_size), (2*cell_size, 2*cell_size), + (cell_size, 2*cell_size)] + gold_corners = np.array(list_gold_corners, dtype=np.float32) + + diamond_corners, diamond_ids, marker_corners, marker_ids = charuco_detector.detectDiamonds(image) + + self.assertEqual(diamond_ids.size, 4) + self.assertEqual(marker_ids.size, 4) + for i in range(0, 4): + self.assertEqual(diamond_ids[0][0][i], i) + np.testing.assert_allclose(gold_corners, np.array(diamond_corners, dtype=np.float32).reshape(-1, 2), 0.01, 0.1) + # check no segfault when cameraMatrix or distCoeffs are not initialized def test_charuco_no_segfault_params(self): dictionary = cv.aruco.getPredefinedDictionary(cv.aruco.DICT_4X4_1000) diff --git a/modules/objdetect/src/aruco/charuco_detector.cpp b/modules/objdetect/src/aruco/charuco_detector.cpp index c39c8b5cc7..b6d3b81964 100644 --- a/modules/objdetect/src/aruco/charuco_detector.cpp +++ b/modules/objdetect/src/aruco/charuco_detector.cpp @@ -315,7 +315,7 @@ void CharucoDetector::detectBoard(InputArray image, OutputArray charucoCorners, } void CharucoDetector::detectDiamonds(InputArray image, OutputArrayOfArrays _diamondCorners, OutputArray _diamondIds, - InputOutputArrayOfArrays inMarkerCorners, InputOutputArrayOfArrays inMarkerIds) const { + InputOutputArrayOfArrays inMarkerCorners, InputOutputArray inMarkerIds) const { CV_Assert(getBoard().getChessboardSize() == Size(3, 3)); CV_Assert((inMarkerCorners.empty() && inMarkerIds.empty() && !image.empty()) || (inMarkerCorners.total() == inMarkerIds.total())); From 0866a135c6902be74635ec9c213f5dc01f1de16f Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Thu, 15 Jun 2023 13:17:25 +0300 Subject: [PATCH 035/105] Git rid of unsafe raw pointers to Mat object. --- modules/calib3d/src/usac/degeneracy.cpp | 30 +++++--- modules/calib3d/src/usac/dls_solver.cpp | 13 ++-- modules/calib3d/src/usac/essential_solver.cpp | 11 +-- modules/calib3d/src/usac/estimator.cpp | 71 ++++++++++--------- .../calib3d/src/usac/fundamental_solver.cpp | 30 ++++---- .../calib3d/src/usac/homography_solver.cpp | 41 +++++++---- modules/calib3d/src/usac/pnp_solver.cpp | 36 ++++++---- 7 files changed, 141 insertions(+), 91 deletions(-) diff --git a/modules/calib3d/src/usac/degeneracy.cpp b/modules/calib3d/src/usac/degeneracy.cpp index a6e028d5f3..6ccf2bb497 100644 --- a/modules/calib3d/src/usac/degeneracy.cpp +++ b/modules/calib3d/src/usac/degeneracy.cpp @@ -8,12 +8,14 @@ namespace cv { namespace usac { class EpipolarGeometryDegeneracyImpl : public EpipolarGeometryDegeneracy { private: - const Mat * points_mat; - const float * const points; // i-th row xi1 yi1 xi2 yi2 + Mat points_mat; const int min_sample_size; public: explicit EpipolarGeometryDegeneracyImpl (const Mat &points_, int sample_size_) : - points_mat(&points_), points ((float*) points_mat->data), min_sample_size (sample_size_) {} + points_mat(points_), min_sample_size (sample_size_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } /* * Do oriented constraint to verify if epipolar geometry is in front or behind the camera. * Return: true if all points are in front of the camers w.r.t. tested epipolar geometry - satisfies constraint. @@ -26,6 +28,7 @@ public: const Vec3d ep = Utils::getRightEpipole(F_); const auto * const e = ep.val; // of size 3x1 const auto * const F = (double *) F_.data; + const float * points = points_mat.ptr(); // without loss of generality, let the first point in sample be in front of the camera. int pt = 4*sample[0]; @@ -67,16 +70,19 @@ Ptr EpipolarGeometryDegeneracy::create (const Mat &p class HomographyDegeneracyImpl : public HomographyDegeneracy { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; const float TOLERANCE = 2 * FLT_EPSILON; // 2 from area of triangle public: explicit HomographyDegeneracyImpl (const Mat &points_) : - points_mat(&points_), points ((float *)points_mat->data) {} + points_mat(points_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } inline bool isSampleGood (const std::vector &sample) const override { const int smpl1 = 4*sample[0], smpl2 = 4*sample[1], smpl3 = 4*sample[2], smpl4 = 4*sample[3]; // planar correspondences must lie on the same side of any line from two points in sample + const float * points = points_mat.ptr(); const float x1 = points[smpl1], y1 = points[smpl1+1], X1 = points[smpl1+2], Y1 = points[smpl1+3]; const float x2 = points[smpl2], y2 = points[smpl2+1], X2 = points[smpl2+2], Y2 = points[smpl2+3]; const float x3 = points[smpl3], y3 = points[smpl3+1], X3 = points[smpl3+2], Y3 = points[smpl3+3]; @@ -188,8 +194,7 @@ private: const Ptr quality; const Ptr f_error; Ptr h_repr_quality; - const float * const points; - const Mat * points_mat; + Mat points_mat; const Ptr h_reproj_error; Ptr f_non_min_solver; Ptr h_non_min_solver; @@ -215,7 +220,7 @@ public: FundamentalDegeneracyImpl (int state, const Ptr &quality_, const Mat &points_, int sample_size_, int plane_and_parallax_iters, double homography_threshold_, double f_inlier_thr_sqr, const Mat true_K1_, const Mat true_K2_) : - rng (state), quality(quality_), f_error(quality_->getErrorFnc()), points((float *) points_.data), points_mat(&points_), + rng (state), quality(quality_), f_error(quality_->getErrorFnc()), points_mat(points_), h_reproj_error(ReprojectionErrorForward::create(points_)), ep_deg (points_, sample_size_), homography_threshold (homography_threshold_), points_size (quality_->getPointsSize()), @@ -330,7 +335,7 @@ public: bool recoverIfDegenerate (const std::vector &sample, const Mat &F_best, const Score &F_best_score, Mat &non_degenerate_model, Score &non_degenerate_model_score) override { const auto swapF = [&] (const Mat &_F, const Score &_score) { - const auto non_min_solver = EpipolarNonMinimalSolver::create(*points_mat, true); + const auto non_min_solver = EpipolarNonMinimalSolver::create(points_mat, true); if (! optimizeF(_F, _score, non_degenerate_model, non_degenerate_model_score)) { _F.copyTo(non_degenerate_model); non_degenerate_model_score = _score; @@ -401,6 +406,7 @@ public: return false; num_models_used_so_far = 0; // reset estimation of lambda for plane-and-parallax int max_iters = max_iters_pl_par, non_planar_support = 0, iters = 0; + const float * points = points_mat.ptr(); std::vector F_good; const double CLOSE_THR = 1.0; for (; iters < max_iters; iters++) { @@ -454,6 +460,7 @@ public: return true; } bool calibDegensac (const Matx33d &H, Mat &F_new, Score &F_new_score, int non_planar_support_degen_F, const Score &F_degen_score) { + const float * points = points_mat.ptr(); if (! is_principal_pt_set) { // estimate principal points from coordinates float px1 = 0, py1 = 0, px2 = 0, py2 = 0; @@ -606,6 +613,7 @@ public: } private: bool getH (const Matx33d &A, const Vec3d &e_prime, int smpl1, int smpl2, int smpl3, Matx33d &H) { + const float * points = points_mat.ptr(); Vec3d p1(points[smpl1 ], points[smpl1+1], 1), p2(points[smpl2 ], points[smpl2+1], 1), p3(points[smpl3 ], points[smpl3+1], 1); Vec3d P1(points[smpl1+2], points[smpl1+3], 1), P2(points[smpl2+2], points[smpl2+3], 1), P3(points[smpl3+2], points[smpl3+3], 1); const Matx33d M (p1[0], p1[1], 1, p2[0], p2[1], 1, p3[0], p3[1], 1); @@ -684,4 +692,4 @@ public: Ptr EssentialDegeneracy::create (const Mat &points_, int sample_size_) { return makePtr(points_, sample_size_); } -}} \ No newline at end of file +}} diff --git a/modules/calib3d/src/usac/dls_solver.cpp b/modules/calib3d/src/usac/dls_solver.cpp index 43b0fc2919..ca6a664504 100644 --- a/modules/calib3d/src/usac/dls_solver.cpp +++ b/modules/calib3d/src/usac/dls_solver.cpp @@ -49,13 +49,15 @@ namespace cv { namespace usac { class DLSPnPImpl : public DLSPnP { #if defined(HAVE_LAPACK) || defined(HAVE_EIGEN) private: - const Mat * points_mat, * calib_norm_points_mat; + Mat points_mat, calib_norm_points_mat; const Matx33d K; - const float * const calib_norm_points, * const points; public: explicit DLSPnPImpl (const Mat &points_, const Mat &calib_norm_points_, const Mat &K_) - : points_mat(&points_), calib_norm_points_mat(&calib_norm_points_), K(K_), - calib_norm_points((float*)calib_norm_points_mat->data), points((float*)points_mat->data) {} + : points_mat(points_), calib_norm_points_mat(calib_norm_points_), K(K_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + CV_DbgAssert(!calib_norm_points_mat.empty() && calib_norm_points_mat.isContinuous()); + } #else public: explicit DLSPnPImpl (const Mat &, const Mat &, const Mat &) {} @@ -88,7 +90,8 @@ public: // Compute V*W*b with the rotation parameters factored out. This is the // translation parameterized by the 9 entries of the rotation matrix. Matx translation_factor = Matx::zeros(); - + const float * points = points_mat.ptr(); + const float * calib_norm_points = calib_norm_points_mat.ptr(); for (int i = 0; i < sample_number; i++) { const int idx_world = 5 * sample[i], idx_calib = 3 * sample[i]; Vec3d normalized_feature_pos(calib_norm_points[idx_calib], diff --git a/modules/calib3d/src/usac/essential_solver.cpp b/modules/calib3d/src/usac/essential_solver.cpp index e2b59a8c49..6dad2a4b88 100644 --- a/modules/calib3d/src/usac/essential_solver.cpp +++ b/modules/calib3d/src/usac/essential_solver.cpp @@ -23,14 +23,17 @@ namespace cv { namespace usac { class EssentialMinimalSolver5ptsImpl : public EssentialMinimalSolver5pts { private: // Points must be calibrated K^-1 x - const Mat * points_mat; - const float * const pts; + const Mat points_mat; const bool use_svd, is_nister; public: explicit EssentialMinimalSolver5ptsImpl (const Mat &points_, bool use_svd_=false, bool is_nister_=false) : - points_mat(&points_), pts((float*)points_mat->data), use_svd(use_svd_), is_nister(is_nister_) {} + points_mat(points_), use_svd(use_svd_), is_nister(is_nister_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } int estimate (const std::vector &sample, std::vector &models) const override { + const float * pts = points_mat.ptr(); // (1) Extract 4 null vectors from linear equations of epipolar constraint std::vector coefficients(45); // 5 pts=rows, 9 columns auto *coefficients_ = &coefficients[0]; @@ -343,4 +346,4 @@ Ptr EssentialMinimalSolver5pts::create (const Mat &points_, bool use_svd, bool is_nister) { return makePtr(points_, use_svd, is_nister); } -}} \ No newline at end of file +}} diff --git a/modules/calib3d/src/usac/estimator.cpp b/modules/calib3d/src/usac/estimator.cpp index f68ebf4cef..262d9196a2 100644 --- a/modules/calib3d/src/usac/estimator.cpp +++ b/modules/calib3d/src/usac/estimator.cpp @@ -216,19 +216,18 @@ Ptr PnPEstimator::create (const Ptr &min_solver_, // Symmetric Reprojection Error class ReprojectionErrorSymmetricImpl : public ReprojectionErrorSymmetric { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; float m11, m12, m13, m21, m22, m23, m31, m32, m33; float minv11, minv12, minv13, minv21, minv22, minv23, minv31, minv32, minv33; std::vector errors; public: explicit ReprojectionErrorSymmetricImpl (const Mat &points_) - : points_mat(&points_), points ((float *) points_.data) + : points_mat(points_) , m11(0), m12(0), m13(0), m21(0), m22(0), m23(0), m31(0), m32(0), m33(0) , minv11(0), minv12(0), minv13(0), minv21(0), minv22(0), minv23(0), minv31(0), minv32(0), minv33(0) , errors(points_.rows) { - CV_DbgAssert(points); + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); } inline void setModelParameters(const Mat& model) override @@ -250,6 +249,7 @@ public: } inline float getError (int idx) const override { idx *= 4; + const float * points = points_mat.ptr(); const float x1=points[idx], y1=points[idx+1], x2=points[idx+2], y2=points[idx+3]; const float est_z2 = 1 / (m31 * x1 + m32 * y1 + m33), dx2 = x2 - (m11 * x1 + m12 * y1 + m13) * est_z2, @@ -261,7 +261,8 @@ public: } const std::vector &getErrors (const Mat &model) override { setModelParameters(model); - for (int point_idx = 0; point_idx < points_mat->rows; point_idx++) { + const float * points = points_mat.ptr(); + for (int point_idx = 0; point_idx < points_mat.rows; point_idx++) { const int smpl = 4*point_idx; const float x1=points[smpl], y1=points[smpl+1], x2=points[smpl+2], y2=points[smpl+3]; const float est_z2 = 1 / (m31 * x1 + m32 * y1 + m33), @@ -283,17 +284,16 @@ ReprojectionErrorSymmetric::create(const Mat &points) { // Forward Reprojection Error class ReprojectionErrorForwardImpl : public ReprojectionErrorForward { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; float m11, m12, m13, m21, m22, m23, m31, m32, m33; std::vector errors; public: explicit ReprojectionErrorForwardImpl (const Mat &points_) - : points_mat(&points_), points ((float *)points_.data) + : points_mat(points_) , m11(0), m12(0), m13(0), m21(0), m22(0), m23(0), m31(0), m32(0), m33(0) , errors(points_.rows) { - CV_DbgAssert(points); + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); } inline void setModelParameters(const Mat& model) override @@ -308,6 +308,7 @@ public: } inline float getError (int idx) const override { idx *= 4; + const float * points = points_mat.ptr(); const float x1 = points[idx], y1 = points[idx+1], x2 = points[idx+2], y2 = points[idx+3]; const float est_z2 = 1 / (m31 * x1 + m32 * y1 + m33), dx2 = x2 - (m11 * x1 + m12 * y1 + m13) * est_z2, @@ -316,7 +317,8 @@ public: } const std::vector &getErrors (const Mat &model) override { setModelParameters(model); - for (int point_idx = 0; point_idx < points_mat->rows; point_idx++) { + const float * points = points_mat.ptr(); + for (int point_idx = 0; point_idx < points_mat.rows; point_idx++) { const int smpl = 4*point_idx; const float x1=points[smpl], y1=points[smpl+1], x2=points[smpl+2], y2=points[smpl+3]; const float est_z2 = 1 / (m31 * x1 + m32 * y1 + m33), @@ -334,17 +336,16 @@ ReprojectionErrorForward::create(const Mat &points) { class SampsonErrorImpl : public SampsonError { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; float m11, m12, m13, m21, m22, m23, m31, m32, m33; std::vector errors; public: explicit SampsonErrorImpl (const Mat &points_) - : points_mat(&points_), points ((float *) points_.data) + : points_mat(points_) , m11(0), m12(0), m13(0), m21(0), m22(0), m23(0), m31(0), m32(0), m33(0) , errors(points_.rows) { - CV_DbgAssert(points); + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); } inline void setModelParameters(const Mat& model) override @@ -370,6 +371,7 @@ public: */ inline float getError (int idx) const override { idx *= 4; + const float * points = points_mat.ptr(); const float x1=points[idx], y1=points[idx+1], x2=points[idx+2], y2=points[idx+3]; const float F_pt1_x = m11 * x1 + m12 * y1 + m13, F_pt1_y = m21 * x1 + m22 * y1 + m23; @@ -381,7 +383,8 @@ public: } const std::vector &getErrors (const Mat &model) override { setModelParameters(model); - for (int point_idx = 0; point_idx < points_mat->rows; point_idx++) { + const float * points = points_mat.ptr(); + for (int point_idx = 0; point_idx < points_mat.rows; point_idx++) { const int smpl = 4*point_idx; const float x1=points[smpl], y1=points[smpl+1], x2=points[smpl+2], y2=points[smpl+3]; const float F_pt1_x = m11 * x1 + m12 * y1 + m13, @@ -402,17 +405,16 @@ SampsonError::create(const Mat &points) { class SymmetricGeometricDistanceImpl : public SymmetricGeometricDistance { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; float m11, m12, m13, m21, m22, m23, m31, m32, m33; std::vector errors; public: explicit SymmetricGeometricDistanceImpl (const Mat &points_) - : points_mat(&points_), points ((float *) points_.data) + : points_mat(points_) , m11(0), m12(0), m13(0), m21(0), m22(0), m23(0), m31(0), m32(0), m33(0) , errors(points_.rows) { - CV_DbgAssert(points); + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); } inline void setModelParameters(const Mat& model) override @@ -428,6 +430,7 @@ public: inline float getError (int idx) const override { idx *= 4; + const float * points = points_mat.ptr(); const float x1=points[idx], y1=points[idx+1], x2=points[idx+2], y2=points[idx+3]; // pt2^T * E, line 1 = [l1 l2] const float l1 = x2 * m11 + y2 * m21 + m31, @@ -443,7 +446,8 @@ public: } const std::vector &getErrors (const Mat &model) override { setModelParameters(model); - for (int point_idx = 0; point_idx < points_mat->rows; point_idx++) { + const float * points = points_mat.ptr(); + for (int point_idx = 0; point_idx < points_mat.rows; point_idx++) { const int smpl = 4*point_idx; const float x1=points[smpl], y1=points[smpl+1], x2=points[smpl+2], y2=points[smpl+3]; const float l1 = x2 * m11 + y2 * m21 + m31, t1 = m11 * x1 + m12 * y1 + m13, @@ -462,17 +466,16 @@ SymmetricGeometricDistance::create(const Mat &points) { class ReprojectionErrorPmatrixImpl : public ReprojectionErrorPmatrix { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; float p11, p12, p13, p14, p21, p22, p23, p24, p31, p32, p33, p34; std::vector errors; public: explicit ReprojectionErrorPmatrixImpl (const Mat &points_) - : points_mat(&points_), points ((float *) points_.data) + : points_mat(points_) , p11(0), p12(0), p13(0), p14(0), p21(0), p22(0), p23(0), p24(0), p31(0), p32(0), p33(0), p34(0) , errors(points_.rows) { - CV_DbgAssert(points); + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); } @@ -489,6 +492,7 @@ public: inline float getError (int idx) const override { idx *= 5; + const float * points = points_mat.ptr(); const float u = points[idx ], v = points[idx+1], x = points[idx+2], y = points[idx+3], z = points[idx+4]; const float depth = 1 / (p31 * x + p32 * y + p33 * z + p34); @@ -498,7 +502,8 @@ public: } const std::vector &getErrors (const Mat &model) override { setModelParameters(model); - for (int point_idx = 0; point_idx < points_mat->rows; point_idx++) { + const float * points = points_mat.ptr(); + for (int point_idx = 0; point_idx < points_mat.rows; point_idx++) { const int smpl = 5*point_idx; const float u = points[smpl ], v = points[smpl+1], x = points[smpl+2], y = points[smpl+3], z = points[smpl+4]; @@ -523,17 +528,16 @@ private: * m21 m22 m23 * 0 0 1 */ - const Mat * points_mat; - const float * const points; + Mat points_mat; float m11, m12, m13, m21, m22, m23; std::vector errors; public: explicit ReprojectionDistanceAffineImpl (const Mat &points_) - : points_mat(&points_), points ((float *) points_.data) + : points_mat(points_) , m11(0), m12(0), m13(0), m21(0), m22(0), m23(0) , errors(points_.rows) { - CV_DbgAssert(points); + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); } inline void setModelParameters(const Mat& model) override @@ -547,13 +551,15 @@ public: } inline float getError (int idx) const override { idx *= 4; + const float * points = points_mat.ptr(); const float x1=points[idx], y1=points[idx+1], x2=points[idx+2], y2=points[idx+3]; const float dx2 = x2 - (m11 * x1 + m12 * y1 + m13), dy2 = y2 - (m21 * x1 + m22 * y1 + m23); return dx2 * dx2 + dy2 * dy2; } const std::vector &getErrors (const Mat &model) override { setModelParameters(model); - for (int point_idx = 0; point_idx < points_mat->rows; point_idx++) { + const float * points = points_mat.ptr(); + for (int point_idx = 0; point_idx < points_mat.rows; point_idx++) { const int smpl = 4*point_idx; const float x1=points[smpl], y1=points[smpl+1], x2=points[smpl+2], y2=points[smpl+3]; const float dx2 = x2 - (m11 * x1 + m12 * y1 + m13), dy2 = y2 - (m21 * x1 + m22 * y1 + m23); @@ -570,13 +576,14 @@ ReprojectionErrorAffine::create(const Mat &points) { ////////////////////////////////////// NORMALIZING TRANSFORMATION ///////////////////////// class NormTransformImpl : public NormTransform { private: - const float * const points; + Mat points_mat; public: - explicit NormTransformImpl (const Mat &points_) : points((float*)points_.data) {} + explicit NormTransformImpl (const Mat &points_) : points_mat(points_) {} // Compute normalized points and transformation matrices. void getNormTransformation (Mat& norm_points, const std::vector &sample, int sample_size, Matx33d &T1, Matx33d &T2) const override { + const float * points = points_mat.ptr(); double mean_pts1_x = 0, mean_pts1_y = 0, mean_pts2_x = 0, mean_pts2_y = 0; // find average of each coordinate of points. diff --git a/modules/calib3d/src/usac/fundamental_solver.cpp b/modules/calib3d/src/usac/fundamental_solver.cpp index 502c3c8c62..4083b76102 100644 --- a/modules/calib3d/src/usac/fundamental_solver.cpp +++ b/modules/calib3d/src/usac/fundamental_solver.cpp @@ -12,17 +12,20 @@ namespace cv { namespace usac { class FundamentalMinimalSolver7ptsImpl: public FundamentalMinimalSolver7pts { private: - const Mat * points_mat; // pointer to OpenCV Mat - const float * const points; // pointer to points_mat->data for faster data access + Mat points_mat; const bool use_ge; public: explicit FundamentalMinimalSolver7ptsImpl (const Mat &points_, bool use_ge_) : - points_mat (&points_), points ((float *) points_mat->data), use_ge(use_ge_) {} + points_mat (points_), use_ge(use_ge_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } int estimate (const std::vector &sample, std::vector &models) const override { const int m = 7, n = 9; // rows, cols std::vector a(63); // m*n auto * a_ = &a[0]; + const float * points = points_mat.ptr(); for (int i = 0; i < m; i++ ) { const int smpl = 4*sample[i]; @@ -158,17 +161,19 @@ Ptr FundamentalMinimalSolver7pts::create(const Mat class FundamentalMinimalSolver8ptsImpl : public FundamentalMinimalSolver8pts { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; public: explicit FundamentalMinimalSolver8ptsImpl (const Mat &points_) : - points_mat (&points_), points ((float*) points_mat->data) - { CV_DbgAssert(points); } + points_mat (points_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } int estimate (const std::vector &sample, std::vector &models) const override { const int m = 8, n = 9; // rows, cols std::vector a(72); // m*n auto * a_ = &a[0]; + const float * points = points_mat.ptr(); for (int i = 0; i < m; i++ ) { const int smpl = 4*sample[i]; @@ -233,18 +238,19 @@ Ptr FundamentalMinimalSolver8pts::create(const Mat class EpipolarNonMinimalSolverImpl : public EpipolarNonMinimalSolver { private: - const Mat * points_mat; + Mat points_mat; const bool do_norm; Matx33d _T1, _T2; Ptr normTr = nullptr; bool enforce_rank = true, is_fundamental, use_ge; public: explicit EpipolarNonMinimalSolverImpl (const Mat &points_, const Matx33d &T1, const Matx33d &T2, bool use_ge_) - : points_mat(&points_), do_norm(false), _T1(T1), _T2(T2), use_ge(use_ge_) { - is_fundamental = true; + : points_mat(points_), do_norm(false), _T1(T1), _T2(T2), is_fundamental(true), use_ge(use_ge_) { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); } explicit EpipolarNonMinimalSolverImpl (const Mat &points_, bool is_fundamental_) : - points_mat(&points_), do_norm(is_fundamental_), use_ge(false) { + points_mat(points_), do_norm(is_fundamental_), use_ge(false) { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); is_fundamental = is_fundamental_; if (is_fundamental) normTr = NormTransform::create(points_); @@ -259,7 +265,7 @@ public: Mat norm_points; if (do_norm) normTr->getNormTransformation(norm_points, sample, sample_size, T1, T2); - const auto * const norm_pts = do_norm ? (float *) norm_points.data : (float *) points_mat->data; + const float * const norm_pts = do_norm ? norm_points.ptr() : points_mat.ptr(); if (use_ge) { double a[8]; diff --git a/modules/calib3d/src/usac/homography_solver.cpp b/modules/calib3d/src/usac/homography_solver.cpp index ba5e233d22..3be5f2dc20 100644 --- a/modules/calib3d/src/usac/homography_solver.cpp +++ b/modules/calib3d/src/usac/homography_solver.cpp @@ -11,14 +11,17 @@ namespace cv { namespace usac { class HomographyMinimalSolver4ptsImpl : public HomographyMinimalSolver4pts { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; const bool use_ge; public: explicit HomographyMinimalSolver4ptsImpl (const Mat &points_, bool use_ge_) : - points_mat(&points_), points ((float*) points_mat->data), use_ge(use_ge_) {} + points_mat(points_), use_ge(use_ge_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } int estimate (const std::vector& sample, std::vector &models) const override { + const float * points = points_mat.ptr(); int m = 8, n = 9; std::vector A(72, 0); int cnt = 0; @@ -80,15 +83,21 @@ Ptr HomographyMinimalSolver4pts::create(const Mat & class HomographyNonMinimalSolverImpl : public HomographyNonMinimalSolver { private: - const Mat * points_mat; + Mat points_mat; const bool do_norm, use_ge; Ptr normTr; Matx33d _T1, _T2; public: explicit HomographyNonMinimalSolverImpl (const Mat &norm_points_, const Matx33d &T1, const Matx33d &T2, bool use_ge_) : - points_mat(&norm_points_), do_norm(false), use_ge(use_ge_), _T1(T1), _T2(T2) {} + points_mat(norm_points_), do_norm(false), use_ge(use_ge_), _T1(T1), _T2(T2) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } explicit HomographyNonMinimalSolverImpl (const Mat &points_, bool use_ge_) : - points_mat(&points_), do_norm(true), use_ge(use_ge_), normTr (NormTransform::create(points_)) {} + points_mat(points_), do_norm(true), use_ge(use_ge_), normTr (NormTransform::create(points_)) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } int estimate (const std::vector &sample, int sample_size, std::vector &models, const std::vector &weights) const override { @@ -99,7 +108,7 @@ public: Mat norm_points_; if (do_norm) normTr->getNormTransformation(norm_points_, sample, sample_size, T1, T2); - const auto * const npts = do_norm ? (float *) norm_points_.data : (float *) points_mat->data; + const float * const npts = do_norm ? norm_points_.ptr() : points_mat.ptr(); Mat H; if (use_ge) { @@ -388,11 +397,13 @@ Ptr CovarianceHomographySolver::create (const Mat &p class AffineMinimalSolverImpl : public AffineMinimalSolver { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; public: explicit AffineMinimalSolverImpl (const Mat &points_) : - points_mat(&points_), points((float *) points_mat->data) {} + points_mat(points_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } /* Affine transformation x1 y1 1 0 0 0 a u1 @@ -404,6 +415,7 @@ public: */ int estimate (const std::vector &sample, std::vector &models) const override { const int smpl1 = 4*sample[0], smpl2 = 4*sample[1], smpl3 = 4*sample[2]; + const float * points = points_mat.ptr(); const auto x1 = points[smpl1], y1 = points[smpl1+1], u1 = points[smpl1+2], v1 = points[smpl1+3], x2 = points[smpl2], y2 = points[smpl2+1], u2 = points[smpl2+2], v2 = points[smpl2+3], @@ -435,13 +447,14 @@ Ptr AffineMinimalSolver::create(const Mat &points_) { class AffineNonMinimalSolverImpl : public AffineNonMinimalSolver { private: - const Mat * points_mat; + Mat points_mat; Ptr normTr; Matx33d _T1, _T2; bool do_norm; public: explicit AffineNonMinimalSolverImpl (const Mat &points_, InputArray T1, InputArray T2) : - points_mat(&points_) { + points_mat(points_) { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); if (!T1.empty() && !T2.empty()) { do_norm = false; _T1 = T1.getMat(); @@ -460,7 +473,7 @@ public: Mat norm_points_; if (do_norm) normTr->getNormTransformation(norm_points_, sample, sample_size, T1, T2); - const auto * const pts = normTr ? (float *) norm_points_.data : (float *) points_mat->data; + const float * const pts = normTr ? norm_points_.ptr() : points_mat.ptr(); // do Least Squares // Ax = b -> A^T Ax = A^T b @@ -640,4 +653,4 @@ Ptr CovarianceAffineSolver::create (const Mat &points, c Ptr CovarianceAffineSolver::create (const Mat &points) { return makePtr(points); } -}} \ No newline at end of file +}} diff --git a/modules/calib3d/src/usac/pnp_solver.cpp b/modules/calib3d/src/usac/pnp_solver.cpp index 704632a0e8..b7b136d1e2 100644 --- a/modules/calib3d/src/usac/pnp_solver.cpp +++ b/modules/calib3d/src/usac/pnp_solver.cpp @@ -15,15 +15,17 @@ namespace cv { namespace usac { class PnPMinimalSolver6PtsImpl : public PnPMinimalSolver6Pts { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; public: // linear 6 points required (11 equations) int getSampleSize() const override { return 6; } int getMaxNumberOfSolutions () const override { return 1; } explicit PnPMinimalSolver6PtsImpl (const Mat &points_) : - points_mat(&points_), points ((float*)points_mat->data) {} + points_mat(points_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } /* DLT: d x = P X, x = (u, v, 1), X = (X, Y, Z, 1), P = K[R t] @@ -72,7 +74,7 @@ public: int estimate (const std::vector &sample, std::vector &models) const override { std::vector A1 (60, 0), A2(56, 0); // 5x12, 7x8 - + const float * points = points_mat.ptr(); // std::vector A_all(11*12, 0); // int cnt3 = 0; @@ -154,14 +156,17 @@ Ptr PnPMinimalSolver6Pts::create(const Mat &points_) { class PnPNonMinimalSolverImpl : public PnPNonMinimalSolver { private: - const Mat * points_mat; - const float * const points; + Mat points_mat; public: explicit PnPNonMinimalSolverImpl (const Mat &points_) : - points_mat(&points_), points ((float*)points_mat->data){} + points_mat(points_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + } int estimate (const std::vector &sample, int sample_size, - std::vector &models, const std::vector &weights) const override { + std::vector &models, const std::vector &weights) const override { + const float * points = points_mat.ptr(); if (sample_size < 6) return 0; @@ -284,9 +289,8 @@ private: * calibrated normalized points * K^-1 [u v 1]^T / ||K^-1 [u v 1]^T|| */ - const Mat * points_mat, * calib_norm_points_mat; + const Mat points_mat, calib_norm_points_mat; const Matx33d K; - const float * const calib_norm_points, * const points; const double VAL_THR = 1e-4; public: /* @@ -294,8 +298,11 @@ public: * u v x y z. (u,v) is image point, (x y z) is world point */ P3PSolverImpl (const Mat &points_, const Mat &calib_norm_points_, const Mat &K_) : - points_mat(&points_), calib_norm_points_mat(&calib_norm_points_), K(K_), - calib_norm_points((float*)calib_norm_points_mat->data), points((float*)points_mat->data) {} + points_mat(points_), calib_norm_points_mat(calib_norm_points_), K(K_) + { + CV_DbgAssert(!points_mat.empty() && points_mat.isContinuous()); + CV_DbgAssert(!calib_norm_points_mat.empty() && calib_norm_points_mat.isContinuous()); + } int estimate (const std::vector &sample, std::vector &models) const override { /* @@ -303,6 +310,9 @@ public: * http://cmp.felk.cvut.cz/~pajdla/gvg/GVG-2016-Lecture.pdf * pages: 51-59 */ + const float * points = points_mat.ptr(); + const float * calib_norm_points = calib_norm_points_mat.ptr(); + const int idx1 = 5*sample[0], idx2 = 5*sample[1], idx3 = 5*sample[2]; const Vec3d X1 (points[idx1+2], points[idx1+3], points[idx1+4]); const Vec3d X2 (points[idx2+2], points[idx2+3], points[idx2+4]); @@ -396,4 +406,4 @@ public: Ptr P3PSolver::create(const Mat &points_, const Mat &calib_norm_pts, const Mat &K) { return makePtr(points_, calib_norm_pts, K); } -}} \ No newline at end of file +}} From 3b264d5877bbdbe8c45c9e55b7543217351da8b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20Kim=20=28=EA=B9=80=ED=98=95=EC=A4=80=29?= <44695374+thekpaul@users.noreply.github.com> Date: Fri, 23 Jun 2023 17:53:03 +0900 Subject: [PATCH 036/105] Add `pthread.h` Inclusion if `HAVE_PTHREADS_PF` is defined Single-case tested with success on Windows 11 with MinGW-w64 Standalone GCC v13.1.0 while building OpenCV 4.7.0 --- modules/core/src/parallel.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/core/src/parallel.cpp b/modules/core/src/parallel.cpp index dc52820e71..94ea91964e 100644 --- a/modules/core/src/parallel.cpp +++ b/modules/core/src/parallel.cpp @@ -118,6 +118,8 @@ #include #elif defined HAVE_CONCURRENCY #include +#elif defined HAVE_PTHREADS_PF + #include #endif From c0fda696f3305d6cf0bf340f56434a6880139a40 Mon Sep 17 00:00:00 2001 From: TolyaTalamanov Date: Fri, 23 Jun 2023 12:16:21 +0000 Subject: [PATCH 037/105] Apply ov::Core WA --- modules/gapi/src/backends/ov/govbackend.cpp | 42 +++++++++++++++---- modules/gapi/src/backends/ov/util.hpp | 10 +++-- .../gapi/test/infer/gapi_infer_ov_tests.cpp | 11 ++--- 3 files changed, 45 insertions(+), 18 deletions(-) diff --git a/modules/gapi/src/backends/ov/govbackend.cpp b/modules/gapi/src/backends/ov/govbackend.cpp index 9497338021..8b6ded45f1 100644 --- a/modules/gapi/src/backends/ov/govbackend.cpp +++ b/modules/gapi/src/backends/ov/govbackend.cpp @@ -18,6 +18,7 @@ #include #include +#include // getConfigurationParameterBool #if defined(HAVE_TBB) # include // FIXME: drop it from here! @@ -37,11 +38,37 @@ template using QueueClass = cv::gapi::own::concurrent_bounded_queue< using ParamDesc = cv::gapi::ov::detail::ParamDesc; -static ov::Core getCore() { +// NB: Some of OV plugins fail during ov::Core destroying in specific cases. +// Solution is allocate ov::Core in heap and doesn't destroy it, which cause +// leak, but fixes tests on CI. This behaviour is configurable by using +// OPENCV_GAPI_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND=0 +static ov::Core create_OV_Core_pointer() { + // NB: 'delete' is never called + static ov::Core* core = new ov::Core(); + return *core; +} + +static ov::Core create_OV_Core_instance() { static ov::Core core; return core; } +ov::Core cv::gapi::ov::wrap::getCore() { + // NB: to make happy memory leak tools use: + // - OPENCV_GAPI_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND=0 + static bool param_GAPI_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND = + utils::getConfigurationParameterBool( + "OPENCV_GAPI_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND", +#if defined(_WIN32) || defined(__APPLE__) + true +#else + false +#endif + ); + return param_GAPI_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND + ? create_OV_Core_pointer() : create_OV_Core_instance(); +} + static ov::AnyMap toOV(const ParamDesc::PluginConfigT &config) { return {config.begin(), config.end()}; } @@ -175,7 +202,8 @@ struct OVUnit { // FIXME: Can this logic be encapsulated to prevent checking every time? if (cv::util::holds_alternative(params.kind)) { const auto desc = cv::util::get(params.kind); - model = getCore().read_model(desc.model_path, desc.bin_path); + model = cv::gapi::ov::wrap::getCore() + .read_model(desc.model_path, desc.bin_path); GAPI_Assert(model); if (params.num_in == 1u && params.input_names.empty()) { @@ -190,9 +218,8 @@ struct OVUnit { std::ifstream file(cv::util::get(params.kind).blob_path, std::ios_base::in | std::ios_base::binary); GAPI_Assert(file.is_open()); - compiled_model = getCore().import_model(file, - params.device, - toOV(params.config)); + compiled_model = cv::gapi::ov::wrap::getCore() + .import_model(file, params.device, toOV(params.config)); if (params.num_in == 1u && params.input_names.empty()) { params.input_names = { compiled_model.inputs().begin()->get_any_name() }; @@ -205,9 +232,8 @@ struct OVUnit { cv::gimpl::ov::OVCompiled compile() { if (cv::util::holds_alternative(params.kind)) { - compiled_model = getCore().compile_model(model, - params.device, - toOV(params.config)); + compiled_model = cv::gapi::ov::wrap::getCore() + .compile_model(model, params.device, toOV(params.config)); } return {compiled_model}; } diff --git a/modules/gapi/src/backends/ov/util.hpp b/modules/gapi/src/backends/ov/util.hpp index 896e707c24..2c769d9ca5 100644 --- a/modules/gapi/src/backends/ov/util.hpp +++ b/modules/gapi/src/backends/ov/util.hpp @@ -22,15 +22,19 @@ namespace cv { namespace gapi { namespace ov { namespace util { - // NB: These functions are EXPORTed to make them accessible by the // test suite only. GAPI_EXPORTS std::vector to_ocv(const ::ov::Shape &shape); GAPI_EXPORTS int to_ocv(const ::ov::element::Type &type); GAPI_EXPORTS void to_ov(const cv::Mat &mat, ::ov::Tensor &tensor); GAPI_EXPORTS void to_ocv(const ::ov::Tensor &tensor, cv::Mat &mat); - -}}}} +} // namespace util +namespace wrap { +GAPI_EXPORTS ::ov::Core getCore(); +} // namespace wrap +} // namespace ov +} // namespace gapi +} // namespace cv #endif // HAVE_INF_ENGINE && INF_ENGINE_RELEASE >= 2022010000 diff --git a/modules/gapi/test/infer/gapi_infer_ov_tests.cpp b/modules/gapi/test/infer/gapi_infer_ov_tests.cpp index 8916b1cdaf..09b54c1a46 100644 --- a/modules/gapi/test/infer/gapi_infer_ov_tests.cpp +++ b/modules/gapi/test/infer/gapi_infer_ov_tests.cpp @@ -52,11 +52,6 @@ void normAssert(cv::InputArray ref, cv::InputArray test, EXPECT_LE(normInf, lInf) << comment; } -ov::Core getCore() { - static ov::Core core; - return core; -} - // TODO: AGNetGenComp, AGNetTypedComp, AGNetOVComp, AGNetOVCompiled // can be generalized to work with any model and used as parameters for tests. @@ -228,7 +223,8 @@ public: const std::string &bin_path, const std::string &device) : m_device(device) { - m_model = getCore().read_model(xml_path, bin_path); + m_model = cv::gapi::ov::wrap::getCore() + .read_model(xml_path, bin_path); } using PrePostProcessF = std::function; @@ -240,7 +236,8 @@ public: } AGNetOVCompiled compile() { - auto compiled_model = getCore().compile_model(m_model, m_device); + auto compiled_model = cv::gapi::ov::wrap::getCore() + .compile_model(m_model, m_device); return {std::move(compiled_model)}; } From e7501b69eabf6ca2fae6f1a747992418cb24435f Mon Sep 17 00:00:00 2001 From: Alexander Panov Date: Fri, 23 Jun 2023 16:16:22 +0300 Subject: [PATCH 038/105] Merge pull request #23647 from AleksandrPanov:fix_charuco_board_detect Add charuco board check #23647 Added charuco board checking to avoid detection of incorrect board. Fixes #23517 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- .../objdetect/src/aruco/charuco_detector.cpp | 102 +++++++++++++----- .../objdetect/test/test_charucodetection.cpp | 28 +++++ 2 files changed, 106 insertions(+), 24 deletions(-) diff --git a/modules/objdetect/src/aruco/charuco_detector.cpp b/modules/objdetect/src/aruco/charuco_detector.cpp index b6d3b81964..55a7f8d888 100644 --- a/modules/objdetect/src/aruco/charuco_detector.cpp +++ b/modules/objdetect/src/aruco/charuco_detector.cpp @@ -23,6 +23,54 @@ struct CharucoDetector::CharucoDetectorImpl { arucoDetector(_arucoDetector) {} + bool checkBoard(InputArrayOfArrays markerCorners, InputArray markerIds, InputArray charucoCorners, InputArray charucoIds) { + vector mCorners; + markerCorners.getMatVector(mCorners); + Mat mIds = markerIds.getMat(); + + Mat chCorners = charucoCorners.getMat(); + Mat chIds = charucoIds.getMat(); + + vector > nearestMarkerIdx = board.getNearestMarkerIdx(); + vector distance(board.getNearestMarkerIdx().size(), Point2f(0.f, std::numeric_limits::max())); + // distance[i].x: max distance from the i-th charuco corner to charuco corner-forming markers. + // The two charuco corner-forming markers of i-th charuco corner are defined in getNearestMarkerIdx()[i] + // distance[i].y: min distance from the charuco corner to other markers. + for (size_t i = 0ull; i < chIds.total(); i++) { + int chId = chIds.ptr(0)[i]; + Point2f charucoCorner(chCorners.ptr(0)[i]); + for (size_t j = 0ull; j < mIds.total(); j++) { + int idMaker = mIds.ptr(0)[j]; + Point2f centerMarker((mCorners[j].ptr(0)[0] + mCorners[j].ptr(0)[1] + + mCorners[j].ptr(0)[2] + mCorners[j].ptr(0)[3]) / 4.f); + float dist = sqrt(normL2Sqr(centerMarker - charucoCorner)); + // check distance from the charuco corner to charuco corner-forming markers + if (nearestMarkerIdx[chId][0] == idMaker || nearestMarkerIdx[chId][1] == idMaker) { + int nearestCornerId = nearestMarkerIdx[chId][0] == idMaker ? board.getNearestMarkerCorners()[chId][0] : board.getNearestMarkerCorners()[chId][1]; + Point2f nearestCorner = mCorners[j].ptr(0)[nearestCornerId]; + float distToNearest = sqrt(normL2Sqr(nearestCorner - charucoCorner)); + distance[chId].x = max(distance[chId].x, distToNearest); + // check that nearestCorner is nearest point + { + Point2f mid1 = (mCorners[j].ptr(0)[(nearestCornerId + 1) % 4]+nearestCorner)*0.5f; + Point2f mid2 = (mCorners[j].ptr(0)[(nearestCornerId + 3) % 4]+nearestCorner)*0.5f; + float tmpDist = min(sqrt(normL2Sqr(mid1 - charucoCorner)), sqrt(normL2Sqr(mid2 - charucoCorner))); + if (tmpDist < distToNearest) + return false; + } + } + // check distance from the charuco corner to other markers + else + distance[chId].y = min(distance[chId].y, dist); + } + // if distance from the charuco corner to charuco corner-forming markers more then distance from the charuco corner to other markers, + // then a false board is found. + if (distance[chId].x > 0.f && distance[chId].y < std::numeric_limits::max() && distance[chId].x > distance[chId].y) + return false; + } + return true; + } + /** Calculate the maximum window sizes for corner refinement for each charuco corner based on the distance * to their closest markers */ vector getMaximumSubPixWindowSizes(InputArrayOfArrays markerCorners, InputArray markerIds, @@ -246,6 +294,31 @@ struct CharucoDetector::CharucoDetectorImpl { Mat(filteredCharucoIds).copyTo(_filteredCharucoIds); return (int)_filteredCharucoIds.total(); } + + void detectBoard(InputArray image, OutputArray charucoCorners, OutputArray charucoIds, + InputOutputArrayOfArrays markerCorners, InputOutputArray markerIds) { + CV_Assert((markerCorners.empty() && markerIds.empty() && !image.empty()) || (markerCorners.total() == markerIds.total())); + vector> tmpMarkerCorners; + vector tmpMarkerIds; + InputOutputArrayOfArrays _markerCorners = markerCorners.needed() ? markerCorners : tmpMarkerCorners; + InputOutputArray _markerIds = markerIds.needed() ? markerIds : tmpMarkerIds; + + if (markerCorners.empty() && markerIds.empty()) { + vector > rejectedMarkers; + arucoDetector.detectMarkers(image, _markerCorners, _markerIds, rejectedMarkers); + if (charucoParameters.tryRefineMarkers) + arucoDetector.refineDetectedMarkers(image, board, _markerCorners, _markerIds, rejectedMarkers); + } + // if camera parameters are avaible, use approximated calibration + if(!charucoParameters.cameraMatrix.empty()) + interpolateCornersCharucoApproxCalib(_markerCorners, _markerIds, image, charucoCorners, charucoIds); + // else use local homography + else + interpolateCornersCharucoLocalHom(_markerCorners, _markerIds, image, charucoCorners, charucoIds); + // to return a charuco corner, its closest aruco markers should have been detected + filterCornersWithoutMinMarkers(charucoCorners, charucoIds, _markerIds, charucoCorners, charucoIds); +} + }; CharucoDetector::CharucoDetector(const CharucoBoard &board, const CharucoParameters &charucoParams, @@ -288,30 +361,11 @@ void CharucoDetector::setRefineParameters(const RefineParameters& refineParamete void CharucoDetector::detectBoard(InputArray image, OutputArray charucoCorners, OutputArray charucoIds, InputOutputArrayOfArrays markerCorners, InputOutputArray markerIds) const { - CV_Assert((markerCorners.empty() && markerIds.empty() && !image.empty()) || (markerCorners.total() == markerIds.total())); - vector> tmpMarkerCorners; - vector tmpMarkerIds; - InputOutputArrayOfArrays _markerCorners = markerCorners.needed() ? markerCorners : tmpMarkerCorners; - InputOutputArray _markerIds = markerIds.needed() ? markerIds : tmpMarkerIds; - - if (markerCorners.empty() && markerIds.empty()) { - vector > rejectedMarkers; - charucoDetectorImpl->arucoDetector.detectMarkers(image, _markerCorners, _markerIds, rejectedMarkers); - if (charucoDetectorImpl->charucoParameters.tryRefineMarkers) - charucoDetectorImpl->arucoDetector.refineDetectedMarkers(image, charucoDetectorImpl->board, _markerCorners, - _markerIds, rejectedMarkers); + charucoDetectorImpl->detectBoard(image, charucoCorners, charucoIds, markerCorners, markerIds); + if (charucoDetectorImpl->checkBoard(markerCorners, markerIds, charucoCorners, charucoIds) == false) { + charucoCorners.release(); + charucoIds.release(); } - // if camera parameters are avaible, use approximated calibration - if(!charucoDetectorImpl->charucoParameters.cameraMatrix.empty()) - charucoDetectorImpl->interpolateCornersCharucoApproxCalib(_markerCorners, _markerIds, image, charucoCorners, - charucoIds); - // else use local homography - else - charucoDetectorImpl->interpolateCornersCharucoLocalHom(_markerCorners, _markerIds, image, charucoCorners, - charucoIds); - // to return a charuco corner, its closest aruco markers should have been detected - charucoDetectorImpl->filterCornersWithoutMinMarkers(charucoCorners, charucoIds, _markerIds, charucoCorners, - charucoIds); } void CharucoDetector::detectDiamonds(InputArray image, OutputArrayOfArrays _diamondCorners, OutputArray _diamondIds, @@ -416,7 +470,7 @@ void CharucoDetector::detectDiamonds(InputArray image, OutputArrayOfArrays _diam // interpolate the charuco corners of the diamond vector currentMarkerCorners; Mat aux; - detectBoard(grey, currentMarkerCorners, aux, currentMarker, currentMarkerId); + charucoDetectorImpl->detectBoard(grey, currentMarkerCorners, aux, currentMarker, currentMarkerId); // if everything is ok, save the diamond if(currentMarkerCorners.size() > 0ull) { diff --git a/modules/objdetect/test/test_charucodetection.cpp b/modules/objdetect/test/test_charucodetection.cpp index 3a459e11fc..9e561bc40a 100644 --- a/modules/objdetect/test/test_charucodetection.cpp +++ b/modules/objdetect/test/test_charucodetection.cpp @@ -689,4 +689,32 @@ TEST(Charuco, testmatchImagePoints) } } +typedef testing::TestWithParam CharucoBoard; +INSTANTIATE_TEST_CASE_P(/**/, CharucoBoard, testing::Values(Size(3, 2), Size(3, 2), Size(6, 2), Size(2, 6), + Size(3, 4), Size(4, 3), Size(7, 3), Size(3, 7))); +TEST_P(CharucoBoard, testWrongSizeDetection) +{ + cv::Size boardSize = GetParam(); + ASSERT_FALSE(boardSize.width == boardSize.height); + aruco::CharucoBoard board(boardSize, 1.f, 0.5f, aruco::getPredefinedDictionary(aruco::DICT_4X4_50)); + + vector detectedCharucoIds, detectedArucoIds; + vector detectedCharucoCorners; + vector> detectedArucoCorners; + Mat boardImage; + board.generateImage(boardSize*40, boardImage); + + swap(boardSize.width, boardSize.height); + aruco::CharucoDetector detector(aruco::CharucoBoard(boardSize, 1.f, 0.5f, aruco::getPredefinedDictionary(aruco::DICT_4X4_50))); + // try detect board with wrong size + detector.detectBoard(boardImage, detectedCharucoCorners, detectedCharucoIds, detectedArucoCorners, detectedArucoIds); + + // aruco markers must be found + ASSERT_EQ(detectedArucoIds.size(), board.getIds().size()); + ASSERT_EQ(detectedArucoCorners.size(), board.getIds().size()); + // charuco corners should not be found in board with wrong size + ASSERT_TRUE(detectedCharucoCorners.empty()); + ASSERT_TRUE(detectedCharucoIds.empty()); +} + }} // namespace From 29388f80a58b953850f23bb8a0efea557e232b7d Mon Sep 17 00:00:00 2001 From: fengyuentau Date: Fri, 23 Jun 2023 17:10:29 +0800 Subject: [PATCH 039/105] fix overflow --- modules/dnn/src/layers/elementwise_layers.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index b84523eb93..e4375abd6d 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -680,7 +680,14 @@ struct SigmoidFunctor : public BaseDefaultFunctor inline float calculate(float x) const { - return 1.f / (1.f + exp(-x)); + float y; + if (x >= 0) + y = 1.f / (1.f + exp(-x)); + else { + y = exp(x); + y = y / (1 + y); + } + return y; } #ifdef HAVE_HALIDE From aff420329cef3a5385dc0c26415d93544b66688b Mon Sep 17 00:00:00 2001 From: Yuantao Feng Date: Sat, 24 Jun 2023 00:52:04 +0800 Subject: [PATCH 040/105] Merge pull request #23853 from fengyuentau:disable_fp16_warning dnn: disable warning when loading a fp16 model #23853 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/onnx/onnx_graph_simplifier.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp index 120cd936b7..8f9be6e961 100644 --- a/modules/dnn/src/onnx/onnx_graph_simplifier.cpp +++ b/modules/dnn/src/onnx/onnx_graph_simplifier.cpp @@ -1125,7 +1125,7 @@ Mat getMatFromTensor(const opencv_onnx::TensorProto& tensor_proto) else if (datatype == opencv_onnx::TensorProto_DataType_FLOAT16) { // FIXME, for now, we only load FP16 Tensor as FP32 Mat, full support for FP16 is required in the future. - CV_LOG_ONCE_WARNING(NULL, "DNN: load FP16 model as FP32 model, and it takes twice the FP16 RAM requirement."); + CV_LOG_ONCE_INFO(NULL, "DNN: load FP16 model as FP32 model, and it takes twice the FP16 RAM requirement."); // ONNX saves float 16 data in two format: int32 and raw_data. // Link: https://github.com/onnx/onnx/issues/4460#issuecomment-1224373746 From 85ea247cc6fd840ee477d03fc9aa31c8c7f6d9ef Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Fri, 23 Jun 2023 22:04:02 +0300 Subject: [PATCH 041/105] Reworked calibrate.py - Fixed width and height swap in board size - Fixed defaults in command line hint - Fixed board visualization for Charuco case - Used matchImagePoints method to handle partially detected Charuco boards --- samples/python/calibrate.py | 77 +++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/samples/python/calibrate.py b/samples/python/calibrate.py index 9528f67aa4..33c87b6a52 100755 --- a/samples/python/calibrate.py +++ b/samples/python/calibrate.py @@ -16,11 +16,13 @@ default values: -w: 4 -h: 6 -t: chessboard - --square_size: 50 - --marker_size: 25 + --square_size: 10 + --marker_size: 5 --aruco_dict: DICT_4X4_50 --threads: 4 defaults to ../data/left*.jpg + +NOTE: Chessboard size is defined in inner corners. Charuco board size is defined in units. ''' # Python 2/3 compatibility @@ -67,45 +69,38 @@ def main(): marker_size = float(args.get('--marker_size')) aruco_dict_name = str(args.get('--aruco_dict')) - pattern_size = (height, width) + pattern_size = (width, height) if pattern_type == 'chessboard': pattern_points = np.zeros((np.prod(pattern_size), 3), np.float32) pattern_points[:, :2] = np.indices(pattern_size).T.reshape(-1, 2) pattern_points *= square_size - elif pattern_type == 'charucoboard': - pattern_points = np.zeros((np.prod((height-1, width-1)), 3), np.float32) - pattern_points[:, :2] = np.indices((height-1, width-1)).T.reshape(-1, 2) - pattern_points *= square_size - else: - print("unknown pattern") - return None obj_points = [] img_points = [] h, w = cv.imread(img_names[0], cv.IMREAD_GRAYSCALE).shape[:2] # TODO: use imquery call to retrieve results aruco_dicts = { - 'DICT_4X4_50':cv.aruco.DICT_4X4_50, - 'DICT_4X4_100':cv.aruco.DICT_4X4_100, - 'DICT_4X4_250':cv.aruco.DICT_4X4_250, - 'DICT_4X4_1000':cv.aruco.DICT_4X4_1000, - 'DICT_5X5_50':cv.aruco.DICT_5X5_50, - 'DICT_5X5_100':cv.aruco.DICT_5X5_100, - 'DICT_5X5_250':cv.aruco.DICT_5X5_250, - 'DICT_5X5_1000':cv.aruco.DICT_5X5_1000, - 'DICT_6X6_50':cv.aruco.DICT_6X6_50, - 'DICT_6X6_100':cv.aruco.DICT_6X6_100, - 'DICT_6X6_250':cv.aruco.DICT_6X6_250, - 'DICT_6X6_1000':cv.aruco.DICT_6X6_1000, - 'DICT_7X7_50':cv.aruco.DICT_7X7_50, - 'DICT_7X7_100':cv.aruco.DICT_7X7_100, - 'DICT_7X7_250':cv.aruco.DICT_7X7_250, - 'DICT_7X7_1000':cv.aruco.DICT_7X7_1000, - 'DICT_ARUCO_ORIGINAL':cv.aruco.DICT_ARUCO_ORIGINAL, - 'DICT_APRILTAG_16h5':cv.aruco.DICT_APRILTAG_16h5, - 'DICT_APRILTAG_25h9':cv.aruco.DICT_APRILTAG_25h9, - 'DICT_APRILTAG_36h10':cv.aruco.DICT_APRILTAG_36h10, - 'DICT_APRILTAG_36h11':cv.aruco.DICT_APRILTAG_36h11 + 'DICT_4X4_50': cv.aruco.DICT_4X4_50, + 'DICT_4X4_100': cv.aruco.DICT_4X4_100, + 'DICT_4X4_250': cv.aruco.DICT_4X4_250, + 'DICT_4X4_1000': cv.aruco.DICT_4X4_1000, + 'DICT_5X5_50': cv.aruco.DICT_5X5_50, + 'DICT_5X5_100': cv.aruco.DICT_5X5_100, + 'DICT_5X5_250': cv.aruco.DICT_5X5_250, + 'DICT_5X5_1000': cv.aruco.DICT_5X5_1000, + 'DICT_6X6_50': cv.aruco.DICT_6X6_50, + 'DICT_6X6_100': cv.aruco.DICT_6X6_100, + 'DICT_6X6_250': cv.aruco.DICT_6X6_250, + 'DICT_6X6_1000': cv.aruco.DICT_6X6_1000, + 'DICT_7X7_50': cv.aruco.DICT_7X7_50, + 'DICT_7X7_100': cv.aruco.DICT_7X7_100, + 'DICT_7X7_250': cv.aruco.DICT_7X7_250, + 'DICT_7X7_1000': cv.aruco.DICT_7X7_1000, + 'DICT_ARUCO_ORIGINAL': cv.aruco.DICT_ARUCO_ORIGINAL, + 'DICT_APRILTAG_16h5': cv.aruco.DICT_APRILTAG_16h5, + 'DICT_APRILTAG_25h9': cv.aruco.DICT_APRILTAG_25h9, + 'DICT_APRILTAG_36h10': cv.aruco.DICT_APRILTAG_36h10, + 'DICT_APRILTAG_36h11': cv.aruco.DICT_APRILTAG_36h11 } if (aruco_dict_name not in set(aruco_dicts.keys())): @@ -130,19 +125,27 @@ def main(): if found: term = (cv.TERM_CRITERIA_EPS + cv.TERM_CRITERIA_COUNT, 30, 0.1) cv.cornerSubPix(img, corners, (5, 5), (-1, -1), term) + frame_img_points = corners.reshape(-1, 2) + frame_obj_points = pattern_points elif pattern_type == 'charucoboard': - corners, _charucoIds, _markerCorners_svg, _markerIds_svg = charuco_detector.detectBoard(img) - if (len(corners) == (height-1)*(width-1)): + corners, charucoIds, _, _ = charuco_detector.detectBoard(img) + if (len(corners) > 0): + frame_obj_points, frame_img_points = board.matchImagePoints(corners, charucoIds) found = True + else: + found = False else: print("unknown pattern type", pattern_type) return None if debug_dir: vis = cv.cvtColor(img, cv.COLOR_GRAY2BGR) - cv.drawChessboardCorners(vis, pattern_size, corners, found) + if pattern_type == 'chessboard': + cv.drawChessboardCorners(vis, pattern_size, corners, found) + elif pattern_type == 'charucoboard': + cv.aruco.drawDetectedCornersCharuco(vis, corners, charucoIds=charucoIds) _path, name, _ext = splitfn(fn) - outfile = os.path.join(debug_dir, name + '_chess.png') + outfile = os.path.join(debug_dir, name + '_board.png') cv.imwrite(outfile, vis) if not found: @@ -150,7 +153,7 @@ def main(): return None print(' %s... OK' % fn) - return (corners.reshape(-1, 2), pattern_points) + return (frame_img_points, frame_obj_points) threads_num = int(args.get('--threads')) if threads_num <= 1: @@ -177,7 +180,7 @@ def main(): print('') for fn in img_names if debug_dir else []: _path, name, _ext = splitfn(fn) - img_found = os.path.join(debug_dir, name + '_chess.png') + img_found = os.path.join(debug_dir, name + '_board.png') outfile = os.path.join(debug_dir, name + '_undistorted.png') img = cv.imread(img_found) From d17507052ef4973ca56df7d56946d6ad8c69cb4b Mon Sep 17 00:00:00 2001 From: Liutong HAN Date: Wed, 28 Jun 2023 17:12:37 +0800 Subject: [PATCH 042/105] Rewrite SIMD code by using new Universal Intrinsic API. --- modules/core/src/mean.simd.hpp | 66 +++++++++++++++++----------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/modules/core/src/mean.simd.hpp b/modules/core/src/mean.simd.hpp index d94c887223..bb815adc1c 100644 --- a/modules/core/src/mean.simd.hpp +++ b/modules/core/src/mean.simd.hpp @@ -24,7 +24,7 @@ struct SumSqr_SIMD } }; -#if CV_SIMD +#if CV_SIMD || CV_SIMD_SCALABLE template <> struct SumSqr_SIMD @@ -39,37 +39,37 @@ struct SumSqr_SIMD v_int32 v_sum = vx_setzero_s32(); v_int32 v_sqsum = vx_setzero_s32(); - const int len0 = len & -v_uint8::nlanes; + const int len0 = len & -VTraits::vlanes(); while(x < len0) { - const int len_tmp = min(x + 256*v_uint16::nlanes, len0); + const int len_tmp = min(x + 256*VTraits::vlanes(), len0); v_uint16 v_sum16 = vx_setzero_u16(); - for ( ; x < len_tmp; x += v_uint8::nlanes) + for ( ; x < len_tmp; x += VTraits::vlanes()) { v_uint16 v_src0 = vx_load_expand(src0 + x); - v_uint16 v_src1 = vx_load_expand(src0 + x + v_uint16::nlanes); - v_sum16 += v_src0 + v_src1; + v_uint16 v_src1 = vx_load_expand(src0 + x + VTraits::vlanes()); + v_sum16 = v_add(v_sum16, v_add(v_src0, v_src1)); v_int16 v_tmp0, v_tmp1; v_zip(v_reinterpret_as_s16(v_src0), v_reinterpret_as_s16(v_src1), v_tmp0, v_tmp1); - v_sqsum += v_dotprod(v_tmp0, v_tmp0) + v_dotprod(v_tmp1, v_tmp1); + v_sqsum = v_add(v_sqsum, v_add(v_dotprod(v_tmp0, v_tmp0), v_dotprod(v_tmp1, v_tmp1))); } v_uint32 v_half0, v_half1; v_expand(v_sum16, v_half0, v_half1); - v_sum += v_reinterpret_as_s32(v_half0 + v_half1); + v_sum = v_add(v_sum, v_reinterpret_as_s32(v_add(v_half0, v_half1))); } - if (x <= len - v_uint16::nlanes) + if (x <= len - VTraits::vlanes()) { v_uint16 v_src = vx_load_expand(src0 + x); v_uint16 v_half = v_combine_high(v_src, v_src); v_uint32 v_tmp0, v_tmp1; - v_expand(v_src + v_half, v_tmp0, v_tmp1); - v_sum += v_reinterpret_as_s32(v_tmp0); + v_expand(v_add(v_src, v_half), v_tmp0, v_tmp1); + v_sum = v_add(v_sum, v_reinterpret_as_s32(v_tmp0)); v_int16 v_tmp2, v_tmp3; v_zip(v_reinterpret_as_s16(v_src), v_reinterpret_as_s16(v_half), v_tmp2, v_tmp3); - v_sqsum += v_dotprod(v_tmp2, v_tmp2); - x += v_uint16::nlanes; + v_sqsum = v_add(v_sqsum, v_dotprod(v_tmp2, v_tmp2)); + x += VTraits::vlanes(); } if (cn == 1) @@ -79,13 +79,13 @@ struct SumSqr_SIMD } else { - int CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[2 * v_int32::nlanes]; + int CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[2 * VTraits::max_nlanes]; v_store(ar, v_sum); - v_store(ar + v_int32::nlanes, v_sqsum); - for (int i = 0; i < v_int32::nlanes; ++i) + v_store(ar + VTraits::vlanes(), v_sqsum); + for (int i = 0; i < VTraits::vlanes(); ++i) { sum[i % cn] += ar[i]; - sqsum[i % cn] += ar[v_int32::nlanes + i]; + sqsum[i % cn] += ar[VTraits::vlanes() + i]; } } v_cleanup(); @@ -106,37 +106,37 @@ struct SumSqr_SIMD v_int32 v_sum = vx_setzero_s32(); v_int32 v_sqsum = vx_setzero_s32(); - const int len0 = len & -v_int8::nlanes; + const int len0 = len & -VTraits::vlanes(); while (x < len0) { - const int len_tmp = min(x + 256 * v_int16::nlanes, len0); + const int len_tmp = min(x + 256 * VTraits::vlanes(), len0); v_int16 v_sum16 = vx_setzero_s16(); - for (; x < len_tmp; x += v_int8::nlanes) + for (; x < len_tmp; x += VTraits::vlanes()) { v_int16 v_src0 = vx_load_expand(src0 + x); - v_int16 v_src1 = vx_load_expand(src0 + x + v_int16::nlanes); - v_sum16 += v_src0 + v_src1; + v_int16 v_src1 = vx_load_expand(src0 + x + VTraits::vlanes()); + v_sum16 = v_add(v_sum16, v_add(v_src0, v_src1)); v_int16 v_tmp0, v_tmp1; v_zip(v_src0, v_src1, v_tmp0, v_tmp1); - v_sqsum += v_dotprod(v_tmp0, v_tmp0) + v_dotprod(v_tmp1, v_tmp1); + v_sqsum = v_add(v_sqsum, v_add(v_dotprod(v_tmp0, v_tmp0), v_dotprod(v_tmp1, v_tmp1))); } v_int32 v_half0, v_half1; v_expand(v_sum16, v_half0, v_half1); - v_sum += v_half0 + v_half1; + v_sum = v_add(v_sum, v_add(v_half0, v_half1)); } - if (x <= len - v_int16::nlanes) + if (x <= len - VTraits::vlanes()) { v_int16 v_src = vx_load_expand(src0 + x); v_int16 v_half = v_combine_high(v_src, v_src); v_int32 v_tmp0, v_tmp1; - v_expand(v_src + v_half, v_tmp0, v_tmp1); - v_sum += v_tmp0; + v_expand(v_add(v_src, v_half), v_tmp0, v_tmp1); + v_sum = v_add(v_sum, v_tmp0); v_int16 v_tmp2, v_tmp3; v_zip(v_src, v_half, v_tmp2, v_tmp3); - v_sqsum += v_dotprod(v_tmp2, v_tmp2); - x += v_int16::nlanes; + v_sqsum = v_add(v_sqsum, v_dotprod(v_tmp2, v_tmp2)); + x += VTraits::vlanes(); } if (cn == 1) @@ -146,13 +146,13 @@ struct SumSqr_SIMD } else { - int CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[2 * v_int32::nlanes]; + int CV_DECL_ALIGNED(CV_SIMD_WIDTH) ar[2 * VTraits::max_nlanes]; v_store(ar, v_sum); - v_store(ar + v_int32::nlanes, v_sqsum); - for (int i = 0; i < v_int32::nlanes; ++i) + v_store(ar + VTraits::vlanes(), v_sqsum); + for (int i = 0; i < VTraits::vlanes(); ++i) { sum[i % cn] += ar[i]; - sqsum[i % cn] += ar[v_int32::nlanes + i]; + sqsum[i % cn] += ar[VTraits::vlanes() + i]; } } v_cleanup(); From b8b8c7c9e5debd60941376d27c4b0ad90cc1fc7b Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Wed, 28 Jun 2023 12:52:15 +0100 Subject: [PATCH 043/105] Merge pull request #23884 from TolyaTalamanov:at/fix-async-infer-ov-backend G-API: Fix async inference for OpenVINO backend #23884 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/gapi/src/backends/ov/govbackend.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/gapi/src/backends/ov/govbackend.cpp b/modules/gapi/src/backends/ov/govbackend.cpp index 8b6ded45f1..8384a9d188 100644 --- a/modules/gapi/src/backends/ov/govbackend.cpp +++ b/modules/gapi/src/backends/ov/govbackend.cpp @@ -1403,8 +1403,10 @@ cv::gimpl::ov::GOVExecutable::GOVExecutable(const ade::Graph &g, case NodeType::OP: if (this_nh == nullptr) { this_nh = nh; - compiled = const_cast(ovm.metadata(this_nh).get()).compile(); - m_reqPool.reset(new RequestPool(createInferRequests(compiled.compiled_model, 1))); + const auto &unit = ovm.metadata(this_nh).get(); + compiled = const_cast(unit).compile(); + m_reqPool.reset(new RequestPool(createInferRequests( + compiled.compiled_model, unit.params.nireq))); } else util::throw_error(std::logic_error("Multi-node inference is not supported!")); @@ -1436,6 +1438,7 @@ void cv::gimpl::ov::GOVExecutable::run(cv::gimpl::GIslandExecutable::IInput &in if (cv::util::holds_alternative(in_msg)) { + m_reqPool->waitAll(); out.post(cv::gimpl::EndOfStream{}); return; } From f9a59f2592993d3dcc080e495f4f5e02dd8ec7ef Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 27 Jun 2023 20:31:20 +0300 Subject: [PATCH 044/105] Release OpenCV 4.8.0 --- modules/core/include/opencv2/core/version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/version.hpp b/modules/core/include/opencv2/core/version.hpp index 2dec11e20c..d5494f53a8 100644 --- a/modules/core/include/opencv2/core/version.hpp +++ b/modules/core/include/opencv2/core/version.hpp @@ -8,7 +8,7 @@ #define CV_VERSION_MAJOR 4 #define CV_VERSION_MINOR 8 #define CV_VERSION_REVISION 0 -#define CV_VERSION_STATUS "-pre" +#define CV_VERSION_STATUS "" #define CVAUX_STR_EXP(__A) #__A #define CVAUX_STR(__A) CVAUX_STR_EXP(__A) From bca5868817ee9ffdf8ad3f79ae40cb6a0d2ff239 Mon Sep 17 00:00:00 2001 From: Wang Kai Date: Sat, 1 Jul 2023 13:29:02 +0800 Subject: [PATCH 045/105] removing duplicated statement --- modules/calib3d/src/usac/gamma_values.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/calib3d/src/usac/gamma_values.cpp b/modules/calib3d/src/usac/gamma_values.cpp index fa1c03e8ed..191ae38a0f 100644 --- a/modules/calib3d/src/usac/gamma_values.cpp +++ b/modules/calib3d/src/usac/gamma_values.cpp @@ -13,7 +13,6 @@ class GammaValuesImpl : public GammaValues { int max_size_table, DoF; public: GammaValuesImpl (int DoF_, int max_size_table_) { - max_size_table = max_size_table_; max_size_table = max_size_table_; DoF = DoF_; /* From 1d9c0d3e1270880971474b6c421a7a453d19018c Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Thu, 22 Jun 2023 15:48:54 +0300 Subject: [PATCH 046/105] videoio: tests for CAP_IMAGES --- .../include/opencv2/videoio/utils.private.hpp | 15 + modules/videoio/src/cap_images.cpp | 4 +- modules/videoio/test/test_images.cpp | 294 ++++++++++++++++++ modules/videoio/test/test_precomp.hpp | 1 + 4 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 modules/videoio/include/opencv2/videoio/utils.private.hpp create mode 100644 modules/videoio/test/test_images.cpp diff --git a/modules/videoio/include/opencv2/videoio/utils.private.hpp b/modules/videoio/include/opencv2/videoio/utils.private.hpp new file mode 100644 index 0000000000..e331aaf2ac --- /dev/null +++ b/modules/videoio/include/opencv2/videoio/utils.private.hpp @@ -0,0 +1,15 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#ifndef OPENCV_VIDEOIO_UTILS_PRIVATE_HPP +#define OPENCV_VIDEOIO_UTILS_PRIVATE_HPP + +#include "opencv2/core/cvdef.h" +#include + +namespace cv { +CV_EXPORTS std::string icvExtractPattern(const std::string& filename, unsigned *offset); +} + +#endif // OPENCV_VIDEOIO_UTILS_PRIVATE_HPP diff --git a/modules/videoio/src/cap_images.cpp b/modules/videoio/src/cap_images.cpp index b506dd1c06..d667e12090 100644 --- a/modules/videoio/src/cap_images.cpp +++ b/modules/videoio/src/cap_images.cpp @@ -51,8 +51,8 @@ #include "precomp.hpp" #include "opencv2/imgcodecs.hpp" - #include "opencv2/core/utils/filesystem.hpp" +#include "opencv2/videoio/utils.private.hpp" #if 0 #define CV_WARN(message) @@ -200,7 +200,7 @@ bool CvCapture_Images::setProperty(int id, double value) return false; } -static +// static std::string icvExtractPattern(const std::string& filename, unsigned *offset) { size_t len = filename.size(); diff --git a/modules/videoio/test/test_images.cpp b/modules/videoio/test/test_images.cpp new file mode 100644 index 0000000000..ccf507a50d --- /dev/null +++ b/modules/videoio/test/test_images.cpp @@ -0,0 +1,294 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" +#include "opencv2/core/utils/filesystem.hpp" +#include "opencv2/imgcodecs.hpp" +#include "opencv2/videoio/utils.private.hpp" + +using namespace std; + +namespace opencv_test { namespace { + +struct ImageCollection +{ + string dirname; + string base; + string ext; + size_t first_idx; + size_t last_idx; + size_t width; +public: + ImageCollection(const char *dirname_template = "opencv_test_images") + : first_idx(0), last_idx(0), width(0) + { + dirname = cv::tempfile(dirname_template); + cv::utils::fs::createDirectory(dirname); + } + ~ImageCollection() + { + cleanup(); + } + void cleanup() + { + cv::utils::fs::remove_all(dirname); + } + void generate(size_t count, size_t first = 0, size_t width_ = 4, const string & base_ = "test", const string & ext_ = "png") + { + base = base_; + ext = ext_; + first_idx = first; + last_idx = first + count - 1; + width = width_; + for (size_t idx = first_idx; idx <= last_idx; ++idx) + { + const string filename = getFilename(idx); + imwrite(filename, getFrame(idx)); + } + } + string getFilename(size_t idx = 0) const + { + ostringstream buf; + buf << dirname << "/" << base << setw(width) << setfill('0') << idx << "." << ext; + return buf.str(); + } + string getPatternFilename() const + { + ostringstream buf; + buf << dirname << "/" << base << "%0" << width << "d" << "." << ext; + return buf.str(); + } + string getFirstFilename() const + { + return getFilename(first_idx); + } + Mat getFirstFrame() const + { + return getFrame(first_idx); + } + size_t getCount() const + { + return last_idx - first_idx + 1; + } + string getDirname() const + { + return dirname; + } + static Mat getFrame(size_t idx) + { + const int sz = 100; // 100x100 or bigger + Mat res(sz, sz, CV_8UC3, Scalar::all(0)); + circle(res, Point(idx % 100), idx % 50, Scalar::all(255), 2, LINE_8); + return res; + } +}; + +//================================================================================================== + +TEST(videoio_images, basic_read) +{ + ImageCollection col; + col.generate(20); + VideoCapture cap(col.getFirstFilename(), CAP_IMAGES); + ASSERT_TRUE(cap.isOpened()); + size_t idx = 0; + while (cap.isOpened()) // TODO: isOpened is always true, even if there are no more images + { + Mat img; + const bool read_res = cap.read(img); + if (!read_res) + break; + EXPECT_MAT_N_DIFF(img, col.getFrame(idx), 0); + ++idx; + } + EXPECT_EQ(col.getCount(), idx); +} + +TEST(videoio_images, basic_write) +{ + // writer should create files: test0000.png, ... test0019.png + ImageCollection col; + col.generate(1); + VideoWriter wri(col.getFirstFilename(), CAP_IMAGES, 0, 0, col.getFrame(0).size()); + ASSERT_TRUE(wri.isOpened()); + size_t idx = 0; + while (wri.isOpened()) + { + wri << col.getFrame(idx); + Mat actual = imread(col.getFilename(idx)); + EXPECT_MAT_N_DIFF(col.getFrame(idx), actual, 0); + if (++idx >= 20) + break; + } + wri.release(); + ASSERT_FALSE(wri.isOpened()); +} + +TEST(videoio_images, bad) +{ + ImageCollection col; + { + ostringstream buf; buf << col.getDirname() << "/missing0000.png"; + VideoCapture cap(buf.str(), CAP_IMAGES); + EXPECT_FALSE(cap.isOpened()); + Mat img; + EXPECT_FALSE(cap.read(img)); + } +} + +TEST(videoio_images, seek) +{ + // check files: test0005.png, ..., test0024.png + // seek to valid and invalid frame numbers + // position is zero-based: valid frame numbers are 0, ..., 19 + const int count = 20; + ImageCollection col; + col.generate(count, 5); + VideoCapture cap(col.getFirstFilename(), CAP_IMAGES); + ASSERT_TRUE(cap.isOpened()); + EXPECT_EQ((size_t)count, (size_t)cap.get(CAP_PROP_FRAME_COUNT)); + vector positions { count / 2, 0, 1, count - 1, count, count + 100, -1, -100 }; + for (const auto &pos : positions) + { + Mat img; + const bool res = cap.set(CAP_PROP_POS_FRAMES, pos); + if (pos >= count || pos < 0) // invalid position + { +// EXPECT_FALSE(res); // TODO: backend clamps invalid value to valid range, actual result is 'true' + } + else + { + EXPECT_TRUE(res); + EXPECT_GE(1., cap.get(CAP_PROP_POS_AVI_RATIO)); + EXPECT_NEAR((double)pos / (count - 1), cap.get(CAP_PROP_POS_AVI_RATIO), 1e-2); + EXPECT_EQ(pos, static_cast(cap.get(CAP_PROP_POS_FRAMES))); + EXPECT_TRUE(cap.read(img)); + EXPECT_MAT_N_DIFF(img, col.getFrame(col.first_idx + pos), 0); + } + } +} + +TEST(videoio_images, pattern_overflow) +{ + // check files: test0.png, ..., test11.png + ImageCollection col; + col.generate(12, 0, 1); + + { + VideoCapture cap(col.getFirstFilename(), CAP_IMAGES); + ASSERT_TRUE(cap.isOpened()); + for (size_t idx = col.first_idx; idx <= col.last_idx; ++idx) + { + Mat img; + EXPECT_TRUE(cap.read(img)); + EXPECT_MAT_N_DIFF(img, col.getFrame(idx), 0); + } + } + { + VideoCapture cap(col.getPatternFilename(), CAP_IMAGES); + ASSERT_TRUE(cap.isOpened()); + for (size_t idx = col.first_idx; idx <= col.last_idx; ++idx) + { + Mat img; + EXPECT_TRUE(cap.read(img)); + EXPECT_MAT_N_DIFF(img, col.getFrame(idx), 0); + } + } +} + +TEST(videoio_images, pattern_max) +{ + // max supported number width for starting image is 9 digits + // but following images can be read as well + // test999999999.png ; test1000000000.png + ImageCollection col; + col.generate(2, 1000000000 - 1); + { + VideoCapture cap(col.getFirstFilename(), CAP_IMAGES); + ASSERT_TRUE(cap.isOpened()); + Mat img; + EXPECT_TRUE(cap.read(img)); + EXPECT_MAT_N_DIFF(img, col.getFrame(col.first_idx), 0); + EXPECT_TRUE(cap.read(img)); + EXPECT_MAT_N_DIFF(img, col.getFrame(col.first_idx + 1), 0); + } + { + VideoWriter wri(col.getFirstFilename(), CAP_IMAGES, 0, 0, col.getFirstFrame().size()); + ASSERT_TRUE(wri.isOpened()); + Mat img = col.getFrame(0); + wri.write(img); + wri.write(img); + Mat actual; + actual = imread(col.getFilename(col.first_idx)); + EXPECT_MAT_N_DIFF(actual, img, 0); + actual = imread(col.getFilename(col.first_idx)); + EXPECT_MAT_N_DIFF(actual, img, 0); + } +} + +TEST(videoio_images, extract_pattern) +{ + unsigned offset = 0; + + // Min and max values + EXPECT_EQ("%01d.png", cv::icvExtractPattern("0.png", &offset)); + EXPECT_EQ(0u, offset); + EXPECT_EQ("%09d.png", cv::icvExtractPattern("999999999.png", &offset)); + EXPECT_EQ(999999999u, offset); + + // Regular usage - start, end, middle + EXPECT_EQ("abc%04ddef.png", cv::icvExtractPattern("abc0048def.png", &offset)); + EXPECT_EQ(48u, offset); + EXPECT_EQ("%05dabcdef.png", cv::icvExtractPattern("00049abcdef.png", &offset)); + EXPECT_EQ(49u, offset); + EXPECT_EQ("abcdef%06d.png", cv::icvExtractPattern("abcdef000050.png", &offset)); + EXPECT_EQ(50u, offset); + + // Minus handling (should not handle) + EXPECT_EQ("abcdef-%01d.png", cv::icvExtractPattern("abcdef-8.png", &offset)); + EXPECT_EQ(8u, offset); + + // Two numbers (should select first) + // TODO: shouldn't it be last number? + EXPECT_EQ("%01d-abcdef-8.png", cv::icvExtractPattern("7-abcdef-8.png", &offset)); + EXPECT_EQ(7u, offset); + + // Paths (should select filename) + EXPECT_EQ("images005/abcdef%03d.png", cv::icvExtractPattern("images005/abcdef006.png", &offset)); + EXPECT_EQ(6u, offset); + // TODO: fix + // EXPECT_EQ("images03\\abcdef%02d.png", cv::icvExtractPattern("images03\\abcdef04.png", &offset)); + // EXPECT_EQ(4, offset); + EXPECT_EQ("/home/user/test/0/3348/../../3442/./0/1/3/4/5/14304324234/%01d.png", + cv::icvExtractPattern("/home/user/test/0/3348/../../3442/./0/1/3/4/5/14304324234/2.png", &offset)); + EXPECT_EQ(2u, offset); + + // Patterns '%0?[0-9][du]' + EXPECT_EQ("test%d.png", cv::icvExtractPattern("test%d.png", &offset)); + EXPECT_EQ(0u, offset); + EXPECT_EQ("test%0d.png", cv::icvExtractPattern("test%0d.png", &offset)); + EXPECT_EQ(0u, offset); + EXPECT_EQ("test%09d.png", cv::icvExtractPattern("test%09d.png", &offset)); + EXPECT_EQ(0u, offset); + EXPECT_EQ("test%5u.png", cv::icvExtractPattern("test%5u.png", &offset)); + EXPECT_EQ(0u, offset); + + // Invalid arguments + EXPECT_THROW(cv::icvExtractPattern(string(), &offset), cv::Exception); + // TODO: fix? + // EXPECT_EQ(0u, offset); + EXPECT_THROW(cv::icvExtractPattern("test%010d.png", &offset), cv::Exception); + EXPECT_EQ(0u, offset); + EXPECT_THROW(cv::icvExtractPattern("1000000000.png", &offset), cv::Exception); + EXPECT_EQ(0u, offset); + EXPECT_THROW(cv::icvExtractPattern("1.png", NULL), cv::Exception); +} + +// TODO: should writer overwrite files? +// TODO: is clamping good for seeking? +// TODO: missing files? E.g. 3, 4, 6, 7, 8 (should it finish OR jump over OR return empty frame?) +// TODO: non-numbered files (https://github.com/opencv/opencv/pull/23815) +// TODO: when opening with pattern (e.g. test%01d.png), first frame can be only 0 (test0.png) + +}} // opencv_test:::: diff --git a/modules/videoio/test/test_precomp.hpp b/modules/videoio/test/test_precomp.hpp index cffdf2bef4..e9ac63d999 100644 --- a/modules/videoio/test/test_precomp.hpp +++ b/modules/videoio/test/test_precomp.hpp @@ -7,6 +7,7 @@ #include #include "opencv2/ts.hpp" +#include "opencv2/ts/ocl_test.hpp" #include "opencv2/videoio.hpp" #include "opencv2/videoio/registry.hpp" #include "opencv2/core/private.hpp" From 50fc68b9818c7acf3eca44e4676c1d8e4a8c8c46 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Mon, 3 Jul 2023 11:00:30 +0300 Subject: [PATCH 047/105] Update PRs & Issue templates to exclude 3.4 from active development. --- .github/ISSUE_TEMPLATE/bug_report.yml | 4 ++-- .github/ISSUE_TEMPLATE/documentation.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index dd53e9508c..2b856bea24 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -15,12 +15,12 @@ body: Please provide the following system information to help us diagnose the bug. For example: // example for c++ user - OpenCV version: 4.6.0 + OpenCV version: 4.8.0 Operating System / Platform: Ubuntu 20.04 Compiler & compiler version: GCC 9.3.0 // example for python user - OpenCV python version: 4.6.0.66 + OpenCV python version: 4.8.0.74 Operating System / Platform: Ubuntu 20.04 Python version: 3.9.6 validations: diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml index d33c030972..8d4d3e440b 100644 --- a/.github/ISSUE_TEMPLATE/documentation.yml +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -12,7 +12,7 @@ body: attributes: label: Describe the doc issue description: > - Please provide a clear and concise description of what content in https://docs.opencv.org/ is an issue. Note that there are multiple active branches, such as 3.4, 4.x and 5.x, so please specify the branch with the problem. + Please provide a clear and concise description of what content in https://docs.opencv.org/ is an issue. Note that there are multiple active branches, such as 4.x and 5.x, so please specify the branch with the problem. placeholder: | A clear and concise description of what content in https://docs.opencv.org/ is an issue. From e9414169a3824d075ad4939c11a97402ffc5cef1 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Thu, 29 Jun 2023 16:21:28 +0200 Subject: [PATCH 048/105] Fix compilation when HAVE_QUIRC is not set. One variable is unknown while the other one is unused. Fixed build warnings. --- modules/objdetect/test/test_qr_utils.hpp | 3 +++ modules/objdetect/test/test_qrcode.cpp | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/objdetect/test/test_qr_utils.hpp b/modules/objdetect/test/test_qr_utils.hpp index cfbe1a5078..115c767f71 100644 --- a/modules/objdetect/test/test_qr_utils.hpp +++ b/modules/objdetect/test/test_qr_utils.hpp @@ -10,6 +10,9 @@ void check_qr(const string& root, const string& name_current_image, const string const std::vector& corners, const std::vector& decoded_info, const int max_pixel_error, bool isMulti = false) { +#ifndef HAVE_QUIRC + CV_UNUSED(decoded_info); +#endif const std::string dataset_config = findDataFile(root + "dataset_config.json"); FileStorage file_config(dataset_config, FileStorage::READ); ASSERT_TRUE(file_config.isOpened()) << "Can't read validation data: " << dataset_config; diff --git a/modules/objdetect/test/test_qrcode.cpp b/modules/objdetect/test/test_qrcode.cpp index 5e6ec6faf5..9b7d8ceda4 100644 --- a/modules/objdetect/test/test_qrcode.cpp +++ b/modules/objdetect/test/test_qrcode.cpp @@ -374,8 +374,8 @@ TEST_P(Objdetect_QRCode_Multi, regression) qrcode = QRCodeDetectorAruco(); } std::vector corners; -#ifdef HAVE_QUIRC std::vector decoded_info; +#ifdef HAVE_QUIRC std::vector straight_barcode; EXPECT_TRUE(qrcode.detectAndDecodeMulti(src, decoded_info, corners, straight_barcode)); ASSERT_FALSE(corners.empty()); @@ -538,7 +538,6 @@ TEST(Objdetect_QRCode_detect_flipped, regression_23249) for(const auto &flipped_image : flipped_images){ const std::string &image_name = flipped_image.first; - const std::string &expect_msg = flipped_image.second; std::string image_path = findDataFile(root + image_name); Mat src = imread(image_path); @@ -551,6 +550,7 @@ TEST(Objdetect_QRCode_detect_flipped, regression_23249) EXPECT_TRUE(!corners.empty()); std::string decoded_msg; #ifdef HAVE_QUIRC + const std::string &expect_msg = flipped_image.second; EXPECT_NO_THROW(decoded_msg = qrcode.decode(src, corners, straight_barcode)); ASSERT_FALSE(straight_barcode.empty()) << "Can't decode qrimage."; EXPECT_EQ(expect_msg, decoded_msg); From 0661aff4a50ff46c1f437dc3cebbe38145ed351c Mon Sep 17 00:00:00 2001 From: Wang Kai <53281385+kai-waang@users.noreply.github.com> Date: Mon, 3 Jul 2023 17:08:12 +0800 Subject: [PATCH 049/105] Merge pull request #23900 from kai-waang:fixing-typo Fixing typos in usac #23900 Just read and correct some typos in `usac` ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/calib3d/src/usac/degeneracy.cpp | 2 +- modules/calib3d/src/usac/essential_solver.cpp | 2 +- modules/calib3d/src/usac/fundamental_solver.cpp | 4 ++-- modules/calib3d/src/usac/pnp_solver.cpp | 4 ++-- modules/calib3d/src/usac/ransac_solvers.cpp | 2 +- modules/calib3d/src/usac/sampler.cpp | 8 ++++---- modules/calib3d/src/usac/termination.cpp | 2 +- modules/calib3d/src/usac/utils.cpp | 4 ++-- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/calib3d/src/usac/degeneracy.cpp b/modules/calib3d/src/usac/degeneracy.cpp index 6ccf2bb497..bf5ec2b110 100644 --- a/modules/calib3d/src/usac/degeneracy.cpp +++ b/modules/calib3d/src/usac/degeneracy.cpp @@ -112,7 +112,7 @@ public: return false; // Checks if points are not collinear - // If area of triangle constructed with 3 points is less then threshold then points are collinear: + // If area of triangle constructed with 3 points is less than threshold then points are collinear: // |x1 y1 1| |x1 y1 1| // (1/2) det |x2 y2 1| = (1/2) det |x2-x1 y2-y1 0| = det |x2-x1 y2-y1| < 2 * threshold // |x3 y3 1| |x3-x1 y3-y1 0| |x3-x1 y3-y1| diff --git a/modules/calib3d/src/usac/essential_solver.cpp b/modules/calib3d/src/usac/essential_solver.cpp index 6dad2a4b88..504fec6ab5 100644 --- a/modules/calib3d/src/usac/essential_solver.cpp +++ b/modules/calib3d/src/usac/essential_solver.cpp @@ -142,7 +142,7 @@ public: } std::vector c(11), rs; - // filling coefficients of 10-degree polynomial satysfying zero-determinant constraint of essential matrix, ie., det(E) = 0 + // filling coefficients of 10-degree polynomial satisfying zero-determinant constraint of essential matrix, ie., det(E) = 0 // based on "An Efficient Solution to the Five-Point Relative Pose Problem" (David Nister) // same as in five-point.cpp c[10] = (b[0]*b[17]*b[34]+b[26]*b[4]*b[21]-b[26]*b[17]*b[8]-b[13]*b[4]*b[34]-b[0]*b[21]*b[30]+b[13]*b[30]*b[8]); diff --git a/modules/calib3d/src/usac/fundamental_solver.cpp b/modules/calib3d/src/usac/fundamental_solver.cpp index 4083b76102..a5e3b30fba 100644 --- a/modules/calib3d/src/usac/fundamental_solver.cpp +++ b/modules/calib3d/src/usac/fundamental_solver.cpp @@ -438,7 +438,7 @@ public: explicit CovarianceEpipolarSolverImpl (const Mat &points_, bool is_fundamental_) { points_size = points_.rows; is_fundamental = is_fundamental_; - if (is_fundamental) { // normalize image points only for fundmantal matrix + if (is_fundamental) { // normalize image points only for fundamental matrix std::vector sample(points_size); for (int i = 0; i < points_size; i++) sample[i] = i; const Ptr normTr = NormTransform::create(points_); @@ -558,7 +558,7 @@ public: const auto * const pts_ = (float *) calib_points.data; // a few point are enough to test // actually due to Sampson error minimization, the input R,t do not really matter - // for a correct pair there is a sligthly faster convergence + // for a correct pair there is a slightly faster convergence for (int i = 0; i < 3; i++) { // could be 1 point const int rand_inl = 4 * sample[rng.uniform(0, sample_size)]; Vec3d p1 (pts_[rand_inl], pts_[rand_inl+1], 1), p2(pts_[rand_inl+2], pts_[rand_inl+3], 1); diff --git a/modules/calib3d/src/usac/pnp_solver.cpp b/modules/calib3d/src/usac/pnp_solver.cpp index b7b136d1e2..db04770908 100644 --- a/modules/calib3d/src/usac/pnp_solver.cpp +++ b/modules/calib3d/src/usac/pnp_solver.cpp @@ -196,7 +196,7 @@ public: a2[10] = v * Z; a2[11] = v; - // fill covarinace matrix + // fill covariance matrix for (int j = 0; j < 12; j++) for (int z = j; z < 12; z++) AtA[j * 12 + z] += a1[j] * a1[z] + a2[j] * a2[z]; @@ -227,7 +227,7 @@ public: a2[10] = v * weight_Z; a2[11] = v * weight; - // fill covarinace matrix + // fill covariance matrix for (int j = 0; j < 12; j++) for (int z = j; z < 12; z++) AtA[j * 12 + z] += a1[j] * a1[z] + a2[j] * a2[z]; diff --git a/modules/calib3d/src/usac/ransac_solvers.cpp b/modules/calib3d/src/usac/ransac_solvers.cpp index 1a76353331..28620c0b3f 100644 --- a/modules/calib3d/src/usac/ransac_solvers.cpp +++ b/modules/calib3d/src/usac/ransac_solvers.cpp @@ -294,7 +294,7 @@ public: params->getUpperIncompleteOfSigmaQuantile()); break; case ScoreMethod::SCORE_METHOD_LMEDS : quality = LMedsQuality::create(points_size, threshold, error); break; - default: CV_Error(cv::Error::StsNotImplemented, "Score is not imeplemeted!"); + default: CV_Error(cv::Error::StsNotImplemented, "Score is not implemented!"); } const auto is_ge_solver = params->getRansacSolver() == GEM_SOLVER; diff --git a/modules/calib3d/src/usac/sampler.cpp b/modules/calib3d/src/usac/sampler.cpp index 2095ee8b4d..1938bde918 100644 --- a/modules/calib3d/src/usac/sampler.cpp +++ b/modules/calib3d/src/usac/sampler.cpp @@ -62,8 +62,8 @@ Ptr UniformSampler::create(int state, int sample_size_, int poin /////////////////////////////////// PROSAC (SIMPLE) SAMPLER /////////////////////////////////////// /* * PROSAC (simple) sampler does not use array of precalculated T_n (n is subset size) samples, but computes T_n for -* specific n directy in generateSample() function. -* Also, the stopping length (or maximum subset size n*) by default is set to points_size (N) and does not updating +* specific n directly in generateSample() function. +* Also, the stopping length (or maximum subset size n*) by default is set to points_size (N) and does not update * during computation. */ class ProsacSimpleSamplerImpl : public ProsacSimpleSampler { @@ -176,7 +176,7 @@ protected: // In our experiments, the parameter was set to T_N = 200000 int growth_max_samples; - // how many time PROSAC generateSample() was called + // how many times PROSAC generateSample() was called int kth_sample_number; Ptr random_gen; public: @@ -488,7 +488,7 @@ public: points_large_neighborhood_size = 0; - // find indicies of points that have sufficient neighborhood (at least sample_size-1) + // find indices of points that have sufficient neighborhood (at least sample_size-1) for (int pt_idx = 0; pt_idx < points_size; pt_idx++) if ((int)neighborhood_graph->getNeighbors(pt_idx).size() >= sample_size-1) points_large_neighborhood[points_large_neighborhood_size++] = pt_idx; diff --git a/modules/calib3d/src/usac/termination.cpp b/modules/calib3d/src/usac/termination.cpp index 803b060e41..26a9e331ed 100644 --- a/modules/calib3d/src/usac/termination.cpp +++ b/modules/calib3d/src/usac/termination.cpp @@ -19,7 +19,7 @@ public: /* * Get upper bound iterations for any sample number - * n is points size, w is inlier ratio, p is desired probability, k is expceted number of iterations. + * n is points size, w is inlier ratio, p is desired probability, k is expected number of iterations. * 1 - p = (1 - w^n)^k, * k = log_(1-w^n) (1-p) * k = ln (1-p) / ln (1-w^n) diff --git a/modules/calib3d/src/usac/utils.cpp b/modules/calib3d/src/usac/utils.cpp index 5e2206702f..da1956f26c 100644 --- a/modules/calib3d/src/usac/utils.cpp +++ b/modules/calib3d/src/usac/utils.cpp @@ -344,10 +344,10 @@ void Utils::decomposeProjection (const Mat &P, Matx33d &K, Matx33d &R, Vec3d &t, } double Utils::getPoissonCDF (double lambda, int inliers) { - double exp_lamda = exp(-lambda), cdf = exp_lamda, lambda_i_div_fact_i = 1; + double exp_lambda = exp(-lambda), cdf = exp_lambda, lambda_i_div_fact_i = 1; for (int i = 1; i <= inliers; i++) { lambda_i_div_fact_i *= (lambda / i); - cdf += exp_lamda * lambda_i_div_fact_i; + cdf += exp_lambda * lambda_i_div_fact_i; if (fabs(cdf - 1) < DBL_EPSILON) // cdf is almost 1 break; } From a58214f015d5511b0f8c1d0c14b9044f4841d643 Mon Sep 17 00:00:00 2001 From: kallaballa Date: Tue, 4 Jul 2023 07:41:16 +0000 Subject: [PATCH 050/105] use CPACK_PACKAGE_VERSION instead of OPENCV_VCSVERSION for CPACK_PACKAGE_FILE_NAME so that OPENCV_CUSTOM_PACKAGE_INFO actually has full effect --- cmake/OpenCVPackaging.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/OpenCVPackaging.cmake b/cmake/OpenCVPackaging.cmake index 7ce7efa661..2480a1cae8 100644 --- a/cmake/OpenCVPackaging.cmake +++ b/cmake/OpenCVPackaging.cmake @@ -52,8 +52,8 @@ else() set(OPENCV_PACKAGE_ARCH_SUFFIX ${CMAKE_SYSTEM_PROCESSOR}) endif() -set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${OPENCV_VCSVERSION}-${OPENCV_PACKAGE_ARCH_SUFFIX}") -set(CPACK_SOURCE_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${OPENCV_VCSVERSION}-${OPENCV_PACKAGE_ARCH_SUFFIX}") +set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${CPACK_PACKAGE_VERSION}-${OPENCV_PACKAGE_ARCH_SUFFIX}") +set(CPACK_SOURCE_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${CPACK_PACKAGE_VERSION}-${OPENCV_PACKAGE_ARCH_SUFFIX}") #rpm options set(CPACK_RPM_COMPONENT_INSTALL TRUE) From 71796edf95cd9b0f9769fef345d00372ef16c506 Mon Sep 17 00:00:00 2001 From: Berke Date: Tue, 4 Jul 2023 21:18:30 +0300 Subject: [PATCH 051/105] removed trailing semicolon after function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It gives error when building projects with -Wpedantic -Werror error: extra ‘;’ [-Werror=pedantic] Issue ##23916 --- modules/core/include/opencv2/core/cuda.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/cuda.hpp b/modules/core/include/opencv2/core/cuda.hpp index 9c948ce00a..5dca06df98 100644 --- a/modules/core/include/opencv2/core/cuda.hpp +++ b/modules/core/include/opencv2/core/cuda.hpp @@ -577,7 +577,7 @@ CV_EXPORTS_W void ensureSizeIsEnough(int rows, int cols, int type, OutputArray a */ CV_EXPORTS_W GpuMat inline createGpuMatFromCudaMemory(int rows, int cols, int type, size_t cudaMemoryAddress, size_t step = Mat::AUTO_STEP) { return GpuMat(rows, cols, type, reinterpret_cast(cudaMemoryAddress), step); -}; +} /** @overload @param size 2D array size: Size(cols, rows). In the Size() constructor, the number of rows and the number of columns go in the reverse order. @@ -588,7 +588,7 @@ CV_EXPORTS_W GpuMat inline createGpuMatFromCudaMemory(int rows, int cols, int ty */ CV_EXPORTS_W inline GpuMat createGpuMatFromCudaMemory(Size size, int type, size_t cudaMemoryAddress, size_t step = Mat::AUTO_STEP) { return GpuMat(size, type, reinterpret_cast(cudaMemoryAddress), step); -}; +} /** @brief BufferPool for use with CUDA streams From 9f8747512917a74a90b6e6c4def04b7e78765995 Mon Sep 17 00:00:00 2001 From: Avasam Date: Tue, 4 Jul 2023 16:44:32 -0400 Subject: [PATCH 052/105] Fix partially unknown Mat --- .../core/misc/python/package/mat_wrapper/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/modules/core/misc/python/package/mat_wrapper/__init__.py b/modules/core/misc/python/package/mat_wrapper/__init__.py index 7309c32b01..44832e7109 100644 --- a/modules/core/misc/python/package/mat_wrapper/__init__.py +++ b/modules/core/misc/python/package/mat_wrapper/__init__.py @@ -1,12 +1,18 @@ __all__ = [] -import sys import numpy as np import cv2 as cv +from typing import TYPE_CHECKING, Any + +# Type subscription is not possible in python 3.8 +if TYPE_CHECKING: + _NDArray = np.ndarray[Any, np.dtype[np.generic]] +else: + _NDArray = np.ndarray # NumPy documentation: https://numpy.org/doc/stable/user/basics.subclassing.html -class Mat(np.ndarray): +class Mat(_NDArray): ''' cv.Mat wrapper for numpy array. From 32251c9b04a3d84d6bd3ed3918fa9493bc5208d0 Mon Sep 17 00:00:00 2001 From: Avasam Date: Tue, 4 Jul 2023 17:50:33 -0400 Subject: [PATCH 053/105] Add missing properties to error class --- .../src2/typing_stubs_generation/api_refinement.py | 13 +++++++++++-- .../src2/typing_stubs_generation/nodes/__init__.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/api_refinement.py b/modules/python/src2/typing_stubs_generation/api_refinement.py index f23167f914..599830f8d8 100644 --- a/modules/python/src2/typing_stubs_generation/api_refinement.py +++ b/modules/python/src2/typing_stubs_generation/api_refinement.py @@ -3,7 +3,8 @@ __all__ = [ ] from typing import Sequence, Callable -from .nodes import NamespaceNode, FunctionNode, OptionalTypeNode + +from .nodes import NamespaceNode, FunctionNode, OptionalTypeNode, ClassProperty, PrimitiveTypeNode from .ast_utils import find_function_node, SymbolName @@ -11,7 +12,7 @@ def apply_manual_api_refinement(root: NamespaceNode) -> None: # Export OpenCV exception class builtin_exception = root.add_class("Exception") builtin_exception.is_exported = False - root.add_class("error", (builtin_exception, )) + root.add_class("error", (builtin_exception, ), ERROR_CLASS_PROPERTIES) for symbol_name, refine_symbol in NODES_TO_REFINE.items(): refine_symbol(root, symbol_name) @@ -46,3 +47,11 @@ NODES_TO_REFINE = { SymbolName(("cv", ), (), "resize"): make_optional_arg("dsize"), SymbolName(("cv", ), (), "calcHist"): make_optional_arg("mask"), } +ERROR_CLASS_PROPERTIES = ( + ClassProperty("code", PrimitiveTypeNode.int_(), False), + ClassProperty("err", PrimitiveTypeNode.str_(), False), + ClassProperty("file", PrimitiveTypeNode.str_(), False), + ClassProperty("func", PrimitiveTypeNode.str_(), False), + ClassProperty("line", PrimitiveTypeNode.int_(), False), + ClassProperty("msg", PrimitiveTypeNode.str_(), False), +) diff --git a/modules/python/src2/typing_stubs_generation/nodes/__init__.py b/modules/python/src2/typing_stubs_generation/nodes/__init__.py index 0ee1df93d9..82f8df8c92 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/__init__.py +++ b/modules/python/src2/typing_stubs_generation/nodes/__init__.py @@ -7,5 +7,5 @@ from .constant_node import ConstantNode from .type_node import ( TypeNode, OptionalTypeNode, UnionTypeNode, NoneTypeNode, TupleTypeNode, ASTNodeTypeNode, AliasTypeNode, SequenceTypeNode, AnyTypeNode, - AggregatedTypeNode, NDArrayTypeNode, AliasRefTypeNode, + AggregatedTypeNode, NDArrayTypeNode, AliasRefTypeNode, PrimitiveTypeNode ) From 10294a84faca9caf85153023fa56ec94df4e37ed Mon Sep 17 00:00:00 2001 From: Zhang Na Date: Tue, 4 Jul 2023 19:19:05 +0800 Subject: [PATCH 054/105] Fix LoongArch Macro Definition --- modules/core/include/opencv2/core/fast_math.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/fast_math.hpp b/modules/core/include/opencv2/core/fast_math.hpp index 47a2948222..bc7f36264a 100644 --- a/modules/core/include/opencv2/core/fast_math.hpp +++ b/modules/core/include/opencv2/core/fast_math.hpp @@ -354,7 +354,7 @@ CV_INLINE int cvFloor( float value ) #if defined CV__FASTMATH_ENABLE_GCC_MATH_BUILTINS || \ defined CV__FASTMATH_ENABLE_CLANG_MATH_BUILTINS return (int)__builtin_floorf(value); -#elif defined __loongarch +#elif defined __loongarch__ int i; float tmp; __asm__ ("ftintrm.w.s %[tmp], %[in] \n\t" @@ -381,7 +381,7 @@ CV_INLINE int cvCeil( float value ) #if defined CV__FASTMATH_ENABLE_GCC_MATH_BUILTINS || \ defined CV__FASTMATH_ENABLE_CLANG_MATH_BUILTINS return (int)__builtin_ceilf(value); -#elif defined __loongarch +#elif defined __loongarch__ int i; float tmp; __asm__ ("ftintrp.w.s %[tmp], %[in] \n\t" From 2d92f42878f45e023c5ae4e8e9c570283a084f6f Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 5 Jul 2023 16:08:33 +0300 Subject: [PATCH 055/105] Disable finite-math-only option with ENABLE_FAST_MATH=1 case to handle NaN and Inf checks correctly. --- cmake/OpenCVCompilerOptions.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index 3f3358aae5..d4600943fb 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -108,6 +108,7 @@ elseif(CV_ICC) elseif(CV_GCC OR CV_CLANG) if(ENABLE_FAST_MATH) add_extra_compiler_option(-ffast-math) + add_extra_compiler_option(-fno-finite-math-only) endif() endif() From 8931f083623412c55d8a98c775f86193b0d66d96 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Thu, 6 Jul 2023 21:08:17 +0300 Subject: [PATCH 056/105] videoio: fix CAP_IMAGES with non-numbered file --- modules/videoio/src/cap_images.cpp | 65 +++++++++++++++--------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/modules/videoio/src/cap_images.cpp b/modules/videoio/src/cap_images.cpp index 87a0b4e27b..32b180b4e3 100644 --- a/modules/videoio/src/cap_images.cpp +++ b/modules/videoio/src/cap_images.cpp @@ -302,59 +302,58 @@ bool CvCapture_Images::open(const std::string& _filename) if (filename_pattern.empty()) { filename_pattern = _filename; - cv::String filename = _filename.c_str(); - if (!utils::fs::exists(filename)) + if (!utils::fs::exists(filename_pattern)) { + CV_LOG_INFO(NULL, "CAP_IMAGES: File does not exist: " << filename_pattern); + close(); return false; } - if (!haveImageReader(filename)) + if (!haveImageReader(filename_pattern)) { - CV_LOG_INFO(NULL, "CAP_IMAGES: Stop scanning. Can't read image file: " << filename); + CV_LOG_INFO(NULL, "CAP_IMAGES: File is not an image: " << filename_pattern); + close(); + return false; } length = 1; - // grab frame to enable properties retrieval - bool grabRes = grabFrame(); - grabbedInOpen = true; - currentframe = 0; - return grabRes; } - - // determine the length of the sequence - for (length = 0; ;) + else { - cv::String filename = cv::format(filename_pattern.c_str(), (int)(offset + length)); - if (!utils::fs::exists(filename)) + // determine the length of the sequence + for (length = 0; ;) { - if (length == 0 && offset == 0) // allow starting with 0 or 1 + cv::String filename = cv::format(filename_pattern.c_str(), (int)(offset + length)); + if (!utils::fs::exists(filename)) { - offset++; - continue; + if (length == 0 && offset == 0) // allow starting with 0 or 1 + { + offset++; + continue; + } + CV_LOG_INFO(NULL, "CAP_IMAGES: File does not exist: " << filename); + break; } - break; + + if(!haveImageReader(filename)) + { + CV_LOG_INFO(NULL, "CAP_IMAGES: File is not an image: " << filename); + break; + } + + length++; } - if(!haveImageReader(filename)) + if (length == 0) { - CV_LOG_INFO(NULL, "CAP_IMAGES: Stop scanning. Can't read image file: " << filename); - break; + close(); + return false; } - length++; + firstframe = offset; } - - if (length == 0) - { - close(); - return false; - } - - firstframe = offset; - // grab frame to enable properties retrieval - bool grabRes = grabFrame(); + bool grabRes = CvCapture_Images::grabFrame(); grabbedInOpen = true; currentframe = 0; - return grabRes; } From e43bc88fc37d04fbb4fcf5436ca568b0987944b4 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Fri, 7 Jul 2023 17:25:49 +0300 Subject: [PATCH 057/105] videoio: test for V4L using virtual device --- modules/videoio/src/cap_v4l.cpp | 8 +- modules/videoio/test/test_precomp.hpp | 8 ++ modules/videoio/test/test_v4l2.cpp | 116 ++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 modules/videoio/test/test_v4l2.cpp diff --git a/modules/videoio/src/cap_v4l.cpp b/modules/videoio/src/cap_v4l.cpp index a5d8561c6b..2f8b8b3745 100644 --- a/modules/videoio/src/cap_v4l.cpp +++ b/modules/videoio/src/cap_v4l.cpp @@ -1854,8 +1854,12 @@ static inline cv::String capPropertyName(int prop) return "auto wb"; case CAP_PROP_WB_TEMPERATURE: return "wb temperature"; + case CAP_PROP_ORIENTATION_META: + return "orientation meta"; + case CAP_PROP_ORIENTATION_AUTO: + return "orientation auto"; default: - return "unknown"; + return cv::format("unknown (%d)", prop); } } @@ -1970,7 +1974,7 @@ bool CvCaptureCAM_V4L::controlInfo(int property_id, __u32 &_v4l2id, cv::Range &r v4l2_queryctrl queryctrl = v4l2_queryctrl(); queryctrl.id = __u32(v4l2id); if (v4l2id == -1 || !tryIoctl(VIDIOC_QUERYCTRL, &queryctrl)) { - CV_LOG_INFO(NULL, "VIDEOIO(V4L2:" << deviceName << "): property " << capPropertyName(property_id) << " is not supported"); + CV_LOG_INFO(NULL, "VIDEOIO(V4L2:" << deviceName << "): property '" << capPropertyName(property_id) << "' is not supported"); return false; } _v4l2id = __u32(v4l2id); diff --git a/modules/videoio/test/test_precomp.hpp b/modules/videoio/test/test_precomp.hpp index e9ac63d999..c39dc7da13 100644 --- a/modules/videoio/test/test_precomp.hpp +++ b/modules/videoio/test/test_precomp.hpp @@ -5,6 +5,7 @@ #define __OPENCV_TEST_PRECOMP_HPP__ #include +#include #include "opencv2/ts.hpp" #include "opencv2/ts/ocl_test.hpp" @@ -56,6 +57,13 @@ inline std::string fourccToString(int fourcc) return cv::format("%c%c%c%c", fourcc & 255, (fourcc >> 8) & 255, (fourcc >> 16) & 255, (fourcc >> 24) & 255); } +inline std::string fourccToStringSafe(int fourcc) +{ + std::string res = fourccToString(fourcc); + std::replace_if(res.begin(), res.end(), [](uint8_t c){ return c < '0' || c > 'z'; }, '_'); + return res; +} + inline int fourccFromString(const std::string &fourcc) { if (fourcc.size() != 4) return 0; diff --git a/modules/videoio/test/test_v4l2.cpp b/modules/videoio/test/test_v4l2.cpp new file mode 100644 index 0000000000..010013c3a9 --- /dev/null +++ b/modules/videoio/test/test_v4l2.cpp @@ -0,0 +1,116 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +// Reference: https://www.kernel.org/doc/html/v4.8/media/v4l-drivers/vivid.html + +// create 1 virtual device of type CAP (0x1) at /dev/video10 +// sudo modprobe vivid ndevs=1 node_types=0x1 vid_cap_nr=10 +// make sure user have read/write access (e.g. via group 'video') +// $ ls -l /dev/video10 +// crw-rw----+ 1 root video ... /dev/video10 +// set environment variable: +// export OPENCV_TEST_V4L2_VIVID_DEVICE=/dev/video10 +// run v4l2 tests: +// opencv_test_videoio --gtest_filter=*videoio_v4l2* + + +#ifdef HAVE_CAMV4L2 + +#include "test_precomp.hpp" +#include +#include + +using namespace cv; + +namespace opencv_test { namespace { + +struct Format_Channels_Depth +{ + uint32_t pixel_format; + uint8_t channels; + uint8_t depth; + float mul_width; + float mul_height; +}; + +typedef testing::TestWithParam videoio_v4l2; + +TEST_P(videoio_v4l2, formats) +{ + utils::Paths devs = utils::getConfigurationParameterPaths("OPENCV_TEST_V4L2_VIVID_DEVICE"); + if (devs.size() != 1) + { + throw SkipTestException("OPENCV_TEST_V4L2_VIVID_DEVICE is not set"); + } + const string device = devs[0]; + const Size sz(640, 480); + const Format_Channels_Depth params = GetParam(); + + { + // Case with RAW output + VideoCapture cap; + ASSERT_TRUE(cap.open(device, CAP_V4L2)); + // VideoCapture will set device's format automatically, vivid device will accept it + ASSERT_TRUE(cap.set(CAP_PROP_FOURCC, params.pixel_format)); + ASSERT_TRUE(cap.set(CAP_PROP_CONVERT_RGB, false)); + for (size_t idx = 0; idx < 3; ++idx) + { + Mat img; + EXPECT_TRUE(cap.grab()); + EXPECT_TRUE(cap.retrieve(img)); + EXPECT_EQ(Size(sz.width * params.mul_width, sz.height * params.mul_height), img.size()); + EXPECT_EQ(params.channels, img.channels()); + EXPECT_EQ(params.depth, img.depth()); + } + } + { + // case with BGR output + VideoCapture cap; + ASSERT_TRUE(cap.open(device, CAP_V4L2)); + // VideoCapture will set device's format automatically, vivid device will accept it + ASSERT_TRUE(cap.set(CAP_PROP_FOURCC, params.pixel_format)); + for (size_t idx = 0; idx < 3; ++idx) + { + Mat img; + EXPECT_TRUE(cap.grab()); + EXPECT_TRUE(cap.retrieve(img)); + EXPECT_EQ(sz, img.size()); + EXPECT_EQ(3, img.channels()); + EXPECT_EQ(CV_8U, img.depth()); + } + } +} + +vector all_params = { + { V4L2_PIX_FMT_YVU420, 1, CV_8U, 1.f, 1.5f }, + { V4L2_PIX_FMT_YUV420, 1, CV_8U, 1.f, 1.5f }, + { V4L2_PIX_FMT_NV12, 1, CV_8U, 1.f, 1.5f }, + { V4L2_PIX_FMT_NV21, 1, CV_8U, 1.f, 1.5f }, + { V4L2_PIX_FMT_YUV411P, 3, CV_8U, 1.f, 1.f }, +// { V4L2_PIX_FMT_MJPEG, 1, CV_8U, 1.f, 1.f }, +// { V4L2_PIX_FMT_JPEG, 1, CV_8U, 1.f, 1.f }, + { V4L2_PIX_FMT_YUYV, 2, CV_8U, 1.f, 1.f }, + { V4L2_PIX_FMT_UYVY, 2, CV_8U, 1.f, 1.f }, +// { V4L2_PIX_FMT_SBGGR8, 1, CV_8U, 1.f, 1.f }, +// { V4L2_PIX_FMT_SN9C10X, 3, CV_8U, 1.f, 1.f }, +// { V4L2_PIX_FMT_SGBRG8, 1, CV_8U, 1.f, 1.f }, + { V4L2_PIX_FMT_RGB24, 3, CV_8U, 1.f, 1.f }, + { V4L2_PIX_FMT_Y16, 1, CV_16U, 1.f, 1.f }, + { V4L2_PIX_FMT_Y10, 1, CV_16U, 1.f, 1.f }, + { V4L2_PIX_FMT_GREY, 1, CV_8U, 1.f, 1.f }, + { V4L2_PIX_FMT_BGR24, 3, CV_8U, 1.f, 1.f }, + { V4L2_PIX_FMT_XBGR32, 3, CV_8U, 1.f, 1.f }, + { V4L2_PIX_FMT_ABGR32, 3, CV_8U, 1.f, 1.f }, +}; + +inline static std::string param_printer(const testing::TestParamInfo& info) +{ + return fourccToStringSafe(info.param.pixel_format); +} + +INSTANTIATE_TEST_CASE_P(/*videoio_v4l2*/, videoio_v4l2, ValuesIn(all_params), param_printer); + +}} // opencv_test:::: + +#endif // HAVE_CAMV4L2 From 09944a83d9f9970b97e5d71b963276023a5c86f6 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Thu, 6 Jul 2023 19:27:16 +0300 Subject: [PATCH 058/105] build: w/a compiler warnings for GCC 11-12 and Clang 13, reduce build output --- 3rdparty/protobuf/CMakeLists.txt | 1 + cmake/copy_files.cmake | 12 +++++----- modules/calib3d/test/test_solvepnp_ransac.cpp | 4 ++-- modules/core/test/test_mat.cpp | 23 ++++++++++++------- .../include/opencv2/gapi/util/variant.hpp | 9 ++++++++ modules/imgproc/src/floodfill.cpp | 7 ++++++ modules/java/generator/gen_java.py | 4 ++-- modules/ts/include/opencv2/ts/ts_gtest.h | 15 ++++++------ 8 files changed, 50 insertions(+), 25 deletions(-) diff --git a/3rdparty/protobuf/CMakeLists.txt b/3rdparty/protobuf/CMakeLists.txt index e39de9823a..5e8e3a9ed2 100644 --- a/3rdparty/protobuf/CMakeLists.txt +++ b/3rdparty/protobuf/CMakeLists.txt @@ -26,6 +26,7 @@ else() -Wsuggest-override -Winconsistent-missing-override -Wimplicit-fallthrough -Warray-bounds # GCC 9+ + -Wstringop-overflow -Wstringop-overread # GCC 11-12 ) endif() if(CV_ICC) diff --git a/cmake/copy_files.cmake b/cmake/copy_files.cmake index 423f7fff9c..f7e13a45d4 100644 --- a/cmake/copy_files.cmake +++ b/cmake/copy_files.cmake @@ -21,7 +21,7 @@ macro(copy_file_ src dst prefix) endif() if(use_symlink) if(local_update OR NOT IS_SYMLINK "${dst}") - message("${prefix}Symlink: '${dst_name}' ...") + #message("${prefix}Symlink: '${dst_name}' ...") endif() get_filename_component(target_path "${dst}" PATH) file(MAKE_DIRECTORY "${target_path}") @@ -38,7 +38,7 @@ macro(copy_file_ src dst prefix) set(local_update 1) endif() if(local_update) - message("${prefix}Copying: '${dst_name}' ...") + #message("${prefix}Copying: '${dst_name}' ...") configure_file(${src} ${dst} COPYONLY) else() #message("${prefix}Up-to-date: '${dst_name}'") @@ -55,7 +55,7 @@ if(NOT DEFINED COPYLIST_VAR) set(COPYLIST_VAR "COPYLIST") endif() list(LENGTH ${COPYLIST_VAR} __length) -message("${prefix}... ${__length} entries (${COPYLIST_VAR})") +#message("${prefix}... ${__length} entries (${COPYLIST_VAR})") foreach(id ${${COPYLIST_VAR}}) set(src "${${COPYLIST_VAR}_SRC_${id}}") set(dst "${${COPYLIST_VAR}_DST_${id}}") @@ -80,7 +80,7 @@ foreach(id ${${COPYLIST_VAR}}) endif() file(GLOB_RECURSE _files RELATIVE "${src}" ${src_glob}) list(LENGTH _files __length) - message("${prefix} ... directory '.../${src_name2}/${src_name}' with ${__length} files") + #message("${prefix} ... directory '.../${src_name2}/${src_name}' with ${__length} files") foreach(f ${_files}) if(NOT EXISTS "${src}/${f}") message(FATAL_ERROR "COPY ERROR: Source file is missing: ${src}/${f}") @@ -98,12 +98,12 @@ else() endif() if(NOT "${__state}" STREQUAL "${__prev_state}") file(WRITE "${STATE_FILE}" "${__state}") - message("${prefix}Updated!") + #message("${prefix}Updated!") set(update_dephelper 1) endif() if(NOT update_dephelper) - message("${prefix}All files are up-to-date.") + #message("${prefix}All files are up-to-date.") elseif(DEFINED DEPHELPER) file(WRITE "${DEPHELPER}" "") endif() diff --git a/modules/calib3d/test/test_solvepnp_ransac.cpp b/modules/calib3d/test/test_solvepnp_ransac.cpp index 43b90dff92..759b9650a8 100644 --- a/modules/calib3d/test/test_solvepnp_ransac.cpp +++ b/modules/calib3d/test/test_solvepnp_ransac.cpp @@ -1531,8 +1531,8 @@ TEST(Calib3d_SolvePnP, generic) } else { - p3f = p3f_; - p2f = p2f_; + p3f = vector(p3f_.begin(), p3f_.end()); + p2f = vector(p2f_.begin(), p2f_.end()); } vector reprojectionErrors; diff --git a/modules/core/test/test_mat.cpp b/modules/core/test/test_mat.cpp index 2d6019eac4..aea0507149 100644 --- a/modules/core/test/test_mat.cpp +++ b/modules/core/test/test_mat.cpp @@ -18,7 +18,7 @@ class Core_ReduceTest : public cvtest::BaseTest public: Core_ReduceTest() {} protected: - void run( int); + void run( int) CV_OVERRIDE; int checkOp( const Mat& src, int dstType, int opType, const Mat& opRes, int dim ); int checkCase( int srcType, int dstType, int dim, Size sz ); int checkDim( int dim, Size sz ); @@ -495,7 +495,7 @@ public: Core_ArrayOpTest(); ~Core_ArrayOpTest(); protected: - void run(int); + void run(int) CV_OVERRIDE; }; @@ -599,6 +599,11 @@ static void setValue(SparseMat& M, const int* idx, double value, RNG& rng) CV_Error(CV_StsUnsupportedFormat, ""); } +#if defined(__GNUC__) && (__GNUC__ == 11 || __GNUC__ == 12) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" +#endif + template struct InitializerFunctor{ /// Initializer for cv::Mat::forEach test @@ -621,6 +626,11 @@ struct InitializerFunctor5D{ } }; +#if defined(__GNUC__) && (__GNUC__ == 11 || __GNUC__ == 12) +#pragma GCC diagnostic pop +#endif + + template struct EmptyFunctor { @@ -1023,7 +1033,7 @@ class Core_MergeSplitBaseTest : public cvtest::BaseTest protected: virtual int run_case(int depth, size_t channels, const Size& size, RNG& rng) = 0; - virtual void run(int) + virtual void run(int) CV_OVERRIDE { // m is Mat // mv is vector @@ -1068,7 +1078,7 @@ public: ~Core_MergeTest() {} protected: - virtual int run_case(int depth, size_t matCount, const Size& size, RNG& rng) + virtual int run_case(int depth, size_t matCount, const Size& size, RNG& rng) CV_OVERRIDE { const int maxMatChannels = 10; @@ -1126,7 +1136,7 @@ public: ~Core_SplitTest() {} protected: - virtual int run_case(int depth, size_t channels, const Size& size, RNG& rng) + virtual int run_case(int depth, size_t channels, const Size& size, RNG& rng) CV_OVERRIDE { Mat src(size, CV_MAKETYPE(depth, (int)channels)); rng.fill(src, RNG::UNIFORM, 0, 100, true); @@ -1990,7 +2000,6 @@ TEST(Core_InputArray, fetch_MatExpr) } -#ifdef CV_CXX11 class TestInputArrayRangeChecking { static const char *kind2str(cv::_InputArray ia) { @@ -2137,8 +2146,6 @@ TEST(Core_InputArray, range_checking) { TestInputArrayRangeChecking::run(); } -#endif - TEST(Core_Vectors, issue_13078) { diff --git a/modules/gapi/include/opencv2/gapi/util/variant.hpp b/modules/gapi/include/opencv2/gapi/util/variant.hpp index f412110deb..48b55646c5 100644 --- a/modules/gapi/include/opencv2/gapi/util/variant.hpp +++ b/modules/gapi/include/opencv2/gapi/util/variant.hpp @@ -509,6 +509,11 @@ namespace util return v.index() == util::variant::template index_of(); } +#if defined(__GNUC__) && (__GNUC__ == 11 || __GNUC__ == 12) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + template bool operator==(const variant &lhs, const variant &rhs) { @@ -524,6 +529,10 @@ namespace util return (eqs[lhs.index()])(lhs.memory, rhs.memory); } +#if defined(__GNUC__) && (__GNUC__ == 11 || __GNUC__ == 12) +#pragma GCC diagnostic pop +#endif + template bool operator!=(const variant &lhs, const variant &rhs) { diff --git a/modules/imgproc/src/floodfill.cpp b/modules/imgproc/src/floodfill.cpp index 926c48e65d..91beca2a99 100644 --- a/modules/imgproc/src/floodfill.cpp +++ b/modules/imgproc/src/floodfill.cpp @@ -560,8 +560,15 @@ int cv::floodFill( InputOutputArray _image, InputOutputArray _mask, if( depth == CV_8U ) for( i = 0; i < cn; i++ ) { +#if defined(__GNUC__) && (__GNUC__ == 12) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-overflow" +#endif ld_buf.b[i] = saturate_cast(cvFloor(loDiff[i])); ud_buf.b[i] = saturate_cast(cvFloor(upDiff[i])); +#if defined(__GNUC__) && (__GNUC__ == 12) +#pragma GCC diagnostic pop +#endif } else if( depth == CV_32S ) for( i = 0; i < cn; i++ ) diff --git a/modules/java/generator/gen_java.py b/modules/java/generator/gen_java.py index 6f07ad913d..7b301193e5 100755 --- a/modules/java/generator/gen_java.py +++ b/modules/java/generator/gen_java.py @@ -686,7 +686,7 @@ class JavaWrapperGenerator(object): msg = "// Return type '%s' is not supported, skipping the function\n\n" % fi.ctype self.skipped_func_list.append(c_decl + "\n" + msg) j_code.write( " "*4 + msg ) - logging.warning("SKIP:" + c_decl.strip() + "\t due to RET type " + fi.ctype) + logging.info("SKIP:" + c_decl.strip() + "\t due to RET type " + fi.ctype) return for a in fi.args: if a.ctype not in type_dict: @@ -698,7 +698,7 @@ class JavaWrapperGenerator(object): msg = "// Unknown type '%s' (%s), skipping the function\n\n" % (a.ctype, a.out or "I") self.skipped_func_list.append(c_decl + "\n" + msg) j_code.write( " "*4 + msg ) - logging.warning("SKIP:" + c_decl.strip() + "\t due to ARG type " + a.ctype + "/" + (a.out or "I")) + logging.info("SKIP:" + c_decl.strip() + "\t due to ARG type " + a.ctype + "/" + (a.out or "I")) return self.ported_func_list.append(c_decl) diff --git a/modules/ts/include/opencv2/ts/ts_gtest.h b/modules/ts/include/opencv2/ts/ts_gtest.h index b1c6c12152..b6a953592f 100644 --- a/modules/ts/include/opencv2/ts/ts_gtest.h +++ b/modules/ts/include/opencv2/ts/ts_gtest.h @@ -17501,6 +17501,7 @@ CartesianProductHolder2(const Generator1& g1, const Generator2& g2) static_cast >(g2_))); } + CartesianProductHolder2(const CartesianProductHolder2 & other) = default; private: // No implementation - assignment is unsupported. void operator=(const CartesianProductHolder2& other) = delete; @@ -17523,7 +17524,7 @@ CartesianProductHolder3(const Generator1& g1, const Generator2& g2, static_cast >(g2_), static_cast >(g3_))); } - + CartesianProductHolder3(const CartesianProductHolder3 &) = default; private: // No implementation - assignment is unsupported. void operator=(const CartesianProductHolder3& other) = delete; @@ -17549,7 +17550,7 @@ CartesianProductHolder4(const Generator1& g1, const Generator2& g2, static_cast >(g3_), static_cast >(g4_))); } - + CartesianProductHolder4(const CartesianProductHolder4 &) = default; private: // No implementation - assignment is unsupported. void operator=(const CartesianProductHolder4& other) = delete; @@ -17577,7 +17578,7 @@ CartesianProductHolder5(const Generator1& g1, const Generator2& g2, static_cast >(g4_), static_cast >(g5_))); } - + CartesianProductHolder5(const CartesianProductHolder5 &) = default; private: // No implementation - assignment is unsupported. void operator=(const CartesianProductHolder5& other) = delete; @@ -17609,7 +17610,7 @@ CartesianProductHolder6(const Generator1& g1, const Generator2& g2, static_cast >(g5_), static_cast >(g6_))); } - + CartesianProductHolder6(const CartesianProductHolder6 &) = default; private: // No implementation - assignment is unsupported. void operator=(const CartesianProductHolder6& other) = delete; @@ -17644,7 +17645,7 @@ CartesianProductHolder7(const Generator1& g1, const Generator2& g2, static_cast >(g6_), static_cast >(g7_))); } - + CartesianProductHolder7(const CartesianProductHolder7 &) = default; private: // No implementation - assignment is unsupported. void operator=(const CartesianProductHolder7& other) = delete; @@ -17683,7 +17684,7 @@ CartesianProductHolder8(const Generator1& g1, const Generator2& g2, static_cast >(g7_), static_cast >(g8_))); } - + CartesianProductHolder8(const CartesianProductHolder8 &) = default; private: // No implementation - assignment is unsupported. void operator=(const CartesianProductHolder8& other) = delete; @@ -17726,7 +17727,7 @@ CartesianProductHolder9(const Generator1& g1, const Generator2& g2, static_cast >(g8_), static_cast >(g9_))); } - + CartesianProductHolder9(const CartesianProductHolder9 &) = default; private: // No implementation - assignment is unsupported. void operator=(const CartesianProductHolder9& other) = delete; From 8097bdc2f40b1da62d975266b88b54fa55a08056 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Mon, 10 Jul 2023 14:30:44 +0300 Subject: [PATCH 059/105] fix: typing stubs overload presence check --- modules/python/src2/typing_stubs_generation/generation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index f89200e0ed..ea6b64bde0 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -535,7 +535,7 @@ def check_overload_presence(node: Union[NamespaceNode, ClassNode]) -> bool: otherwise. """ for func_node in node.functions.values(): - if len(func_node.overloads): + if len(func_node.overloads) > 1: return True return False From a6df33647719b0f90efb1d52836e8719d83eaede Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Mon, 10 Jul 2023 14:38:15 +0300 Subject: [PATCH 060/105] fix: recursively re-export nested submodules --- .../python/src2/typing_stubs_generation/generation.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index f89200e0ed..536621dd8e 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -621,12 +621,17 @@ def _populate_reexported_symbols(root: NamespaceNode) -> None: # Re-export all submodules to allow referencing symbols in submodules # without submodule import. Example: # `cv2.aruco.ArucoDetector` should be accessible without `import cv2.aruco` - for submodule in root.namespaces.values(): - root.reexported_submodules.append(submodule.export_name) + def _reexport_submodule(ns: NamespaceNode) -> None: + for submodule in ns.namespaces.values(): + ns.reexported_submodules.append(submodule.export_name) + _reexport_submodule(submodule) + + _reexport_submodule(root) # Special cases, symbols defined in possible pure Python submodules should be root.reexported_submodules_symbols["mat_wrapper"].append("Mat") + def _write_reexported_symbols_section(module: NamespaceNode, output_stream: StringIO) -> None: """Write re-export section for the given module. From 986e379f280a79e0f68b848c412a99776026aafc Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Mon, 10 Jul 2023 13:55:54 +0300 Subject: [PATCH 061/105] feat: add matrix type stubs generation Adds missing typing stubs: - Matrix depths: `CV_8U`, `CV_8S` and etc. - Matrix type constants: `CV_8UC1`, `CV_32FC3` and etc. - Matrix type factory functions: `CV_*(channels) -> int` and `CV_MAKETYPE` --- .../typing_stubs_generation/api_refinement.py | 31 ++++++++++++++++++- .../typing_stubs_generation/generation.py | 3 -- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/api_refinement.py b/modules/python/src2/typing_stubs_generation/api_refinement.py index 599830f8d8..f7d3a9dd08 100644 --- a/modules/python/src2/typing_stubs_generation/api_refinement.py +++ b/modules/python/src2/typing_stubs_generation/api_refinement.py @@ -4,11 +4,13 @@ __all__ = [ from typing import Sequence, Callable -from .nodes import NamespaceNode, FunctionNode, OptionalTypeNode, ClassProperty, PrimitiveTypeNode +from .nodes import (NamespaceNode, FunctionNode, OptionalTypeNode, + ClassProperty, PrimitiveTypeNode) from .ast_utils import find_function_node, SymbolName def apply_manual_api_refinement(root: NamespaceNode) -> None: + export_matrix_type_constants(root) # Export OpenCV exception class builtin_exception = root.add_class("Exception") builtin_exception.is_exported = False @@ -17,6 +19,33 @@ def apply_manual_api_refinement(root: NamespaceNode) -> None: refine_symbol(root, symbol_name) +def export_matrix_type_constants(root: NamespaceNode) -> None: + MAX_PREDEFINED_CHANNELS = 4 + + depth_names = ("CV_8U", "CV_8S", "CV_16U", "CV_16S", "CV_32S", + "CV_32F", "CV_64F", "CV_16F") + for depth_value, depth_name in enumerate(depth_names): + # Export depth constants + root.add_constant(depth_name, str(depth_value)) + # Export predefined types + for c in range(MAX_PREDEFINED_CHANNELS): + root.add_constant(f"{depth_name}C{c + 1}", + f"{depth_value + 8 * c}") + # Export type creation function + root.add_function( + f"{depth_name}C", + (FunctionNode.Arg("channels", PrimitiveTypeNode.int_()), ), + FunctionNode.RetType(PrimitiveTypeNode.int_()) + ) + # Export CV_MAKETYPE + root.add_function( + "CV_MAKETYPE", + (FunctionNode.Arg("depth", PrimitiveTypeNode.int_()), + FunctionNode.Arg("channels", PrimitiveTypeNode.int_())), + FunctionNode.RetType(PrimitiveTypeNode.int_()) + ) + + def make_optional_arg(arg_name: str) -> Callable[[NamespaceNode, SymbolName], None]: def _make_optional_arg(root_node: NamespaceNode, function_symbol_name: SymbolName) -> None: diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index f89200e0ed..c377c9e01b 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -103,9 +103,6 @@ def _generate_typing_stubs(root: NamespaceNode, output_path: Path) -> None: _write_reexported_symbols_section(root, output_stream) - # Write constants section, because constants don't impose any dependencies - _generate_section_stub(StubSection("# Constants", ConstantNode), root, - output_stream, 0) # NOTE: Enumerations require special handling, because all enumeration # constants are exposed as module attributes has_enums = _generate_section_stub(StubSection("# Enumerations", EnumerationNode), From 953de60ff0b5008eb9f7ca79c6d6c297cfee5040 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Mon, 10 Jul 2023 18:05:32 +0300 Subject: [PATCH 062/105] feat: update NumPy type to Mat type fail message Output string representation of NumPy array type if it is not convertible to OpenCV Mat type --- modules/python/src2/cv2_convert.cpp | 25 ++++++++++++++++++++++++- modules/python/src2/cv2_util.hpp | 6 +++++- modules/python/test/test_misc.py | 8 ++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/modules/python/src2/cv2_convert.cpp b/modules/python/src2/cv2_convert.cpp index 2e69586f47..20cd957753 100644 --- a/modules/python/src2/cv2_convert.cpp +++ b/modules/python/src2/cv2_convert.cpp @@ -5,6 +5,7 @@ #include "cv2_convert.hpp" #include "cv2_numpy.hpp" +#include "cv2_util.hpp" #include "opencv2/core/utils/logger.hpp" PyTypeObject* pyopencv_Mat_TypePtr = nullptr; @@ -24,6 +25,26 @@ static std::string pycv_dumpArray(const T* arr, int n) return out.str(); } +static inline std::string getArrayTypeName(PyArrayObject* arr) +{ + PyArray_Descr* dtype = PyArray_DESCR(arr); + PySafeObject dtype_str(PyObject_Str(reinterpret_cast(dtype))); + if (!dtype_str) + { + // Fallback to typenum value + return cv::format("%d", PyArray_TYPE(arr)); + } + std::string type_name; + if (!getUnicodeString(dtype_str, type_name)) + { + // Failed to get string from bytes object - clear set TypeError and + // fallback to typenum value + PyErr_Clear(); + return cv::format("%d", PyArray_TYPE(arr)); + } + return type_name; +} + //====================================================================================================================== // --- Mat @@ -102,7 +123,9 @@ bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) } else { - failmsg("%s data type = %d is not supported", info.name, typenum); + const std::string dtype_name = getArrayTypeName(oarr); + failmsg("%s data type = %s is not supported", info.name, + dtype_name.c_str()); return false; } } diff --git a/modules/python/src2/cv2_util.hpp b/modules/python/src2/cv2_util.hpp index 0d27e98825..c15fcac486 100644 --- a/modules/python/src2/cv2_util.hpp +++ b/modules/python/src2/cv2_util.hpp @@ -42,7 +42,7 @@ private: /** * Light weight RAII wrapper for `PyObject*` owning references. - * In comparisson to C++11 `std::unique_ptr` with custom deleter, it provides + * In comparison to C++11 `std::unique_ptr` with custom deleter, it provides * implicit conversion functions that might be useful to initialize it with * Python functions those returns owning references through the `PyObject**` * e.g. `PyErr_Fetch` or directly pass it to functions those want to borrow @@ -70,6 +70,10 @@ public: return &obj_; } + operator bool() { + return static_cast(obj_); + } + PyObject* release() { PyObject* obj = obj_; diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index ec7f44de0f..ed13d66b93 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -219,6 +219,14 @@ class Arguments(NewOpenCVTests): #res6 = cv.utils.dumpInputArray([a, b]) #self.assertEqual(res6, "InputArrayOfArrays: empty()=false kind=0x00050000 flags=0x01050000 total(-1)=2 dims(-1)=1 size(-1)=2x1 type(0)=CV_32FC1 dims(0)=4 size(0)=[2 3 4 5]") + def test_unsupported_numpy_data_types_string_description(self): + for dtype in (object, str, np.complex128): + test_array = np.zeros((4, 4, 3), dtype=dtype) + msg = ".*type = {} is not supported".format(test_array.dtype) + self.assertRaisesRegex( + Exception, msg, cv.utils.dumpInputArray, test_array + ) + def test_20968(self): pixel = np.uint8([[[40, 50, 200]]]) _ = cv.cvtColor(pixel, cv.COLOR_RGB2BGR) # should not raise exception From f89b705049d108608cff8a0a818588c8d195d23b Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 11 Jul 2023 01:09:31 +0000 Subject: [PATCH 063/105] cmake: don't export external target twice --- cmake/OpenCVUtils.cmake | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmake/OpenCVUtils.cmake b/cmake/OpenCVUtils.cmake index 437042958e..a90eb5a5ab 100644 --- a/cmake/OpenCVUtils.cmake +++ b/cmake/OpenCVUtils.cmake @@ -1632,13 +1632,19 @@ function(ocv_add_external_target name inc link def) endif() endfunction() +set(__OPENCV_EXPORTED_EXTERNAL_TARGETS "" CACHE INTERNAL "") function(ocv_install_used_external_targets) if(NOT BUILD_SHARED_LIBS AND NOT (CMAKE_VERSION VERSION_LESS "3.13.0") # upgrade CMake: https://gitlab.kitware.com/cmake/cmake/-/merge_requests/2152 ) foreach(tgt in ${ARGN}) if(tgt MATCHES "^ocv\.3rdparty\.") - install(TARGETS ${tgt} EXPORT OpenCVModules) + list(FIND __OPENCV_EXPORTED_EXTERNAL_TARGETS "${tgt}" _found) + if(_found EQUAL -1) # don't export target twice + install(TARGETS ${tgt} EXPORT OpenCVModules) + list(APPEND __OPENCV_EXPORTED_EXTERNAL_TARGETS "${tgt}") + set(__OPENCV_EXPORTED_EXTERNAL_TARGETS "${__OPENCV_EXPORTED_EXTERNAL_TARGETS}" CACHE INTERNAL "") + endif() endif() endforeach() endif() From 2af6775c14a553ebe6e38f6a8f54b4d023c6ebb0 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Tue, 11 Jul 2023 09:25:53 +0200 Subject: [PATCH 064/105] Merge pull request #23943 from vrabaud:avif_tsan Fix checkSignature not thread safe for AVIF. #23943 A common decoder cannot be shared with checkSignature which is used like a static function (on a static ist of decoders). ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/imgcodecs/src/grfmt_avif.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/imgcodecs/src/grfmt_avif.cpp b/modules/imgcodecs/src/grfmt_avif.cpp index e8d1446cbe..b2830b755a 100644 --- a/modules/imgcodecs/src/grfmt_avif.cpp +++ b/modules/imgcodecs/src/grfmt_avif.cpp @@ -148,11 +148,14 @@ AvifDecoder::~AvifDecoder() { size_t AvifDecoder::signatureLength() const { return kAvifSignatureSize; } bool AvifDecoder::checkSignature(const String &signature) const { - avifDecoderSetIOMemory(decoder_, + avifDecoder *decoder = avifDecoderCreate(); + if (!decoder) return false; + avifDecoderSetIOMemory(decoder, reinterpret_cast(signature.c_str()), signature.size()); - decoder_->io->sizeHint = 1e9; - const avifResult status = avifDecoderParse(decoder_); + decoder->io->sizeHint = 1e9; + const avifResult status = avifDecoderParse(decoder); + avifDecoderDestroy(decoder); return (status == AVIF_RESULT_OK || status == AVIF_RESULT_TRUNCATED_DATA); } From a00818047ff5671586c7b296cac6175b250f85d3 Mon Sep 17 00:00:00 2001 From: Liutong HAN Date: Tue, 11 Jul 2023 16:10:27 +0800 Subject: [PATCH 065/105] =?UTF-8?q?Add=20missing=20=E2=80=9Dv=5Fpopcount?= =?UTF-8?q?=E2=80=9C=20for=20RVV=20and=20enable=20tests.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../include/opencv2/core/hal/intrin_rvv_scalable.hpp | 9 +++++++++ modules/core/test/test_intrin_utils.hpp | 2 ++ 2 files changed, 11 insertions(+) diff --git a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp index 60066ba041..dab82489f8 100644 --- a/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_rvv_scalable.hpp @@ -1651,6 +1651,10 @@ inline v_uint32 v_popcount(const v_uint32& a) { return v_hadd(v_hadd(v_popcount(vreinterpret_u8m1(a)))); } +inline v_uint64 v_popcount(const v_uint64& a) +{ + return v_hadd(v_hadd(v_hadd(v_popcount(vreinterpret_u8m1(a))))); +} inline v_uint8 v_popcount(const v_int8& a) { @@ -1664,6 +1668,11 @@ inline v_uint32 v_popcount(const v_int32& a) { return v_popcount(v_abs(a));\ } +inline v_uint64 v_popcount(const v_int64& a) +{ + // max(0 - a) is used, since v_abs does not support 64-bit integers. + return v_popcount(v_reinterpret_as_u64(vmax(a, v_sub(v_setzero_s64(), a), VTraits::vlanes()))); +} //////////// SignMask //////////// diff --git a/modules/core/test/test_intrin_utils.hpp b/modules/core/test/test_intrin_utils.hpp index 01ca50a26e..481e6bb1f2 100644 --- a/modules/core/test/test_intrin_utils.hpp +++ b/modules/core/test/test_intrin_utils.hpp @@ -2048,6 +2048,7 @@ void test_hal_intrin_uint64() .test_rotate<0>().test_rotate<1>() .test_extract_n<0>().test_extract_n<1>() .test_extract_highest() + .test_popcount() //.test_broadcast_element<0>().test_broadcast_element<1>() ; } @@ -2069,6 +2070,7 @@ void test_hal_intrin_int64() .test_extract_highest() //.test_broadcast_element<0>().test_broadcast_element<1>() .test_cvt64_double() + .test_popcount() ; } From 87f8cdd699288192bb60e68a38485ea362e99359 Mon Sep 17 00:00:00 2001 From: Yusuke Kameda Date: Tue, 11 Jul 2023 19:08:58 +0900 Subject: [PATCH 066/105] doxygen: Fix ImwriteFlags documentation misalignment --- modules/imgcodecs/include/opencv2/imgcodecs.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/imgcodecs/include/opencv2/imgcodecs.hpp b/modules/imgcodecs/include/opencv2/imgcodecs.hpp index c1bdf72291..5e201b52fb 100644 --- a/modules/imgcodecs/include/opencv2/imgcodecs.hpp +++ b/modules/imgcodecs/include/opencv2/imgcodecs.hpp @@ -95,11 +95,11 @@ enum ImwriteFlags { IMWRITE_PNG_STRATEGY = 17, //!< One of cv::ImwritePNGFlags, default is IMWRITE_PNG_STRATEGY_RLE. IMWRITE_PNG_BILEVEL = 18, //!< Binary level PNG, 0 or 1, default is 0. IMWRITE_PXM_BINARY = 32, //!< For PPM, PGM, or PBM, it can be a binary format flag, 0 or 1. Default value is 1. - IMWRITE_EXR_TYPE = (3 << 4) + 0, /* 48 */ //!< override EXR storage type (FLOAT (FP32) is default) - IMWRITE_EXR_COMPRESSION = (3 << 4) + 1, /* 49 */ //!< override EXR compression type (ZIP_COMPRESSION = 3 is default) - IMWRITE_EXR_DWA_COMPRESSION_LEVEL = (3 << 4) + 2, /* 50 */ //!< override EXR DWA compression level (45 is default) + IMWRITE_EXR_TYPE = (3 << 4) + 0 /* 48 */, //!< override EXR storage type (FLOAT (FP32) is default) + IMWRITE_EXR_COMPRESSION = (3 << 4) + 1 /* 49 */, //!< override EXR compression type (ZIP_COMPRESSION = 3 is default) + IMWRITE_EXR_DWA_COMPRESSION_LEVEL = (3 << 4) + 2 /* 50 */, //!< override EXR DWA compression level (45 is default) IMWRITE_WEBP_QUALITY = 64, //!< For WEBP, it can be a quality from 1 to 100 (the higher is the better). By default (without any parameter) and for quality above 100 the lossless compression is used. - IMWRITE_HDR_COMPRESSION = (5 << 4) + 0, /* 80 */ //!< specify HDR compression + IMWRITE_HDR_COMPRESSION = (5 << 4) + 0 /* 80 */, //!< specify HDR compression IMWRITE_PAM_TUPLETYPE = 128,//!< For PAM, sets the TUPLETYPE field to the corresponding string value that is defined for the format IMWRITE_TIFF_RESUNIT = 256,//!< For TIFF, use to specify which DPI resolution unit to set; see libtiff documentation for valid values IMWRITE_TIFF_XDPI = 257,//!< For TIFF, use to specify the X direction DPI From e3c1405254fe170a82ae5625db5c7ed876c313b2 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Tue, 11 Jul 2023 17:05:32 +0300 Subject: [PATCH 067/105] videoio: fix v4l2 test on older platforms (centos) --- modules/videoio/test/test_v4l2.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/modules/videoio/test/test_v4l2.cpp b/modules/videoio/test/test_v4l2.cpp index 010013c3a9..6763e2e33c 100644 --- a/modules/videoio/test/test_v4l2.cpp +++ b/modules/videoio/test/test_v4l2.cpp @@ -21,6 +21,20 @@ #include #include +// workarounds for older versions +#ifndef V4L2_PIX_FMT_Y10 +#define V4L2_PIX_FMT_Y10 v4l2_fourcc('Y', '1', '0', ' ') +#endif +#ifndef V4L2_PIX_FMT_Y12 +#define V4L2_PIX_FMT_Y12 v4l2_fourcc('Y', '1', '2', ' ') +#endif +#ifndef V4L2_PIX_FMT_ABGR32 +#define V4L2_PIX_FMT_ABGR32 v4l2_fourcc('A', 'R', '2', '4') +#endif +#ifndef V4L2_PIX_FMT_XBGR32 +#define V4L2_PIX_FMT_XBGR32 v4l2_fourcc('X', 'R', '2', '4') +#endif + using namespace cv; namespace opencv_test { namespace { From 48c52c8bbb90065c1663eebc8524e3145d3ab9c7 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 11 Jul 2023 16:49:11 +0300 Subject: [PATCH 068/105] Fixed tests execution with Python 2.7 --- modules/python/src2/cv2_util.hpp | 2 +- modules/python/test/test_misc.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/python/src2/cv2_util.hpp b/modules/python/src2/cv2_util.hpp index c15fcac486..a7deb5b575 100644 --- a/modules/python/src2/cv2_util.hpp +++ b/modules/python/src2/cv2_util.hpp @@ -71,7 +71,7 @@ public: } operator bool() { - return static_cast(obj_); + return obj_ != nullptr; } PyObject* release() diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index ed13d66b93..afe3fcc5db 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -223,9 +223,14 @@ class Arguments(NewOpenCVTests): for dtype in (object, str, np.complex128): test_array = np.zeros((4, 4, 3), dtype=dtype) msg = ".*type = {} is not supported".format(test_array.dtype) - self.assertRaisesRegex( - Exception, msg, cv.utils.dumpInputArray, test_array - ) + if sys.version_info[0] < 3: + self.assertRaisesRegexp( + Exception, msg, cv.utils.dumpInputArray, test_array + ) + else: + self.assertRaisesRegex( + Exception, msg, cv.utils.dumpInputArray, test_array + ) def test_20968(self): pixel = np.uint8([[[40, 50, 200]]]) From 7b4b7ceb7eab30d4c2075db1a4f0a5fbc9d5a65e Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 11 Jul 2023 18:39:36 +0300 Subject: [PATCH 069/105] Add Ubuntu 22.04 to CI. --- .github/workflows/PR-4.x.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/PR-4.x.yaml b/.github/workflows/PR-4.x.yaml index 551bade422..307e6c7452 100644 --- a/.github/workflows/PR-4.x.yaml +++ b/.github/workflows/PR-4.x.yaml @@ -15,6 +15,9 @@ jobs: Ubuntu2004-x64: uses: opencv/ci-gha-workflow/.github/workflows/OCV-PR-4.x-U20.yaml@main + Ubuntu2204-x64: + uses: opencv/ci-gha-workflow/.github/workflows/OCV-PR-4.x-U22.yaml@main + Ubuntu2004-x64-CUDA: if: "${{ contains(github.event.pull_request.labels.*.name, 'category: dnn') }} || ${{ contains(github.event.pull_request.labels.*.name, 'category: dnn (onnx)') }}" uses: opencv/ci-gha-workflow/.github/workflows/OCV-PR-4.x-U20-Cuda.yaml@main From cd9f85dbdab16f35aa772fa000d75714b9e0e972 Mon Sep 17 00:00:00 2001 From: Avasam Date: Tue, 11 Jul 2023 12:00:57 -0400 Subject: [PATCH 070/105] Update usages of ConditionalAliasTypeNode following #23838 to use TYPE_CHECKING --- .../misc/python/package/mat_wrapper/__init__.py | 8 ++++---- .../typing_stubs_generation/nodes/type_node.py | 15 +++++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/modules/core/misc/python/package/mat_wrapper/__init__.py b/modules/core/misc/python/package/mat_wrapper/__init__.py index 44832e7109..7cbc0645de 100644 --- a/modules/core/misc/python/package/mat_wrapper/__init__.py +++ b/modules/core/misc/python/package/mat_wrapper/__init__.py @@ -4,15 +4,15 @@ import numpy as np import cv2 as cv from typing import TYPE_CHECKING, Any -# Type subscription is not possible in python 3.8 +# Same as cv2.typing.NumPyArrayGeneric, but avoids circular dependencies if TYPE_CHECKING: - _NDArray = np.ndarray[Any, np.dtype[np.generic]] + _NumPyArrayGeneric = np.ndarray[Any, np.dtype[np.generic]] else: - _NDArray = np.ndarray + _NumPyArrayGeneric = np.ndarray # NumPy documentation: https://numpy.org/doc/stable/user/basics.subclassing.html -class Mat(_NDArray): +class Mat(_NumPyArrayGeneric): ''' cv.Mat wrapper for numpy array. diff --git a/modules/python/src2/typing_stubs_generation/nodes/type_node.py b/modules/python/src2/typing_stubs_generation/nodes/type_node.py index 912adc6954..8af4dd0039 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/type_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/type_node.py @@ -395,9 +395,12 @@ class AliasTypeNode(TypeNode): class ConditionalAliasTypeNode(TypeNode): """Type node representing an alias protected by condition checked in runtime. + For typing-related conditions, prefer using typing.TYPE_CHECKING. For a full explanation, see: + https://github.com/opencv/opencv/pull/23927#discussion_r1256326835 + Example: ```python - if numpy.lib.NumpyVersion(numpy.__version__) > "1.20.0" and sys.version_info >= (3, 9) + if typing.TYPE_CHECKING NumPyArray = numpy.ndarray[typing.Any, numpy.dtype[numpy.generic]] else: NumPyArray = numpy.ndarray @@ -407,10 +410,10 @@ class ConditionalAliasTypeNode(TypeNode): ConditionalAliasTypeNode( "NumPyArray", - 'numpy.lib.NumpyVersion(numpy.__version__) > "1.20.0" and sys.version_info >= (3, 9)', + 'typing.TYPE_CHECKING', NDArrayTypeNode("NumPyArray"), NDArrayTypeNode("NumPyArray", use_numpy_generics=False), - condition_required_imports=("import numpy", "import sys") + condition_required_imports=("import typing",) ) ``` """ @@ -468,14 +471,14 @@ class ConditionalAliasTypeNode(TypeNode): def numpy_array_(cls, ctype_name: str, export_name: Optional[str] = None, shape: Optional[Tuple[int, ...]] = None, dtype: Optional[str] = None): + """Type subscription is not possible in python 3.8 and older numpy versions.""" return cls( ctype_name, - ('numpy.lib.NumpyVersion(numpy.__version__) > "1.20.0" ' - 'and sys.version_info >= (3, 9)'), + "typing.TYPE_CHECKING", NDArrayTypeNode(ctype_name, shape, dtype), NDArrayTypeNode(ctype_name, shape, dtype, use_numpy_generics=False), - condition_required_imports=("import numpy", "import sys") + condition_required_imports=("import typing",) ) From 3f0707234f7115d0248d943417144c9eab268e45 Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Tue, 11 Jul 2023 19:23:12 +0300 Subject: [PATCH 071/105] risc-v: fix unaligned loads and stores --- modules/core/src/matrix_transform.cpp | 8 ++++---- modules/imgproc/src/demosaicing.cpp | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/core/src/matrix_transform.cpp b/modules/core/src/matrix_transform.cpp index 57fd0c6509..7f1043fbbe 100644 --- a/modules/core/src/matrix_transform.cpp +++ b/modules/core/src/matrix_transform.cpp @@ -603,10 +603,10 @@ flipVert( const uchar* src0, size_t sstep, uchar* dst0, size_t dstep, Size size, { for (; i <= size.width - CV_SIMD_WIDTH; i += CV_SIMD_WIDTH) { - v_int32 t0 = vx_load((int*)(src0 + i)); - v_int32 t1 = vx_load((int*)(src1 + i)); - v_store((int*)(dst0 + i), t1); - v_store((int*)(dst1 + i), t0); + v_int32 t0 = v_reinterpret_as_s32(vx_load(src0 + i)); + v_int32 t1 = v_reinterpret_as_s32(vx_load(src1 + i)); + v_store(dst0 + i, v_reinterpret_as_u8(t1)); + v_store(dst1 + i, v_reinterpret_as_u8(t0)); } } #if CV_STRONG_ALIGNMENT diff --git a/modules/imgproc/src/demosaicing.cpp b/modules/imgproc/src/demosaicing.cpp index 27dfc1520c..627c052aea 100644 --- a/modules/imgproc/src/demosaicing.cpp +++ b/modules/imgproc/src/demosaicing.cpp @@ -184,9 +184,9 @@ public: for( ; bayer <= bayer_end - 18; bayer += 14, dst += 14 ) { - v_uint16x8 r0 = v_load((ushort*)bayer); - v_uint16x8 r1 = v_load((ushort*)(bayer+bayer_step)); - v_uint16x8 r2 = v_load((ushort*)(bayer+bayer_step*2)); + v_uint16x8 r0 = v_reinterpret_as_u16(v_load(bayer)); + v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step)); + v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2)); v_uint16x8 b1 = ((r0 << 8) >> 7) + ((r2 << 8) >> 7); v_uint16x8 b0 = v_rotate_right<1>(b1) + b1; @@ -265,9 +265,9 @@ public: for( ; bayer <= bayer_end - 18; bayer += 14, dst += 42 ) { - v_uint16x8 r0 = v_load((ushort*)bayer); - v_uint16x8 r1 = v_load((ushort*)(bayer+bayer_step)); - v_uint16x8 r2 = v_load((ushort*)(bayer+bayer_step*2)); + v_uint16x8 r0 = v_reinterpret_as_u16(v_load(bayer)); + v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step)); + v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2)); v_uint16x8 b1 = (r0 & masklo) + (r2 & masklo); v_uint16x8 nextb1 = v_rotate_right<1>(b1); @@ -398,9 +398,9 @@ public: for( ; bayer <= bayer_end - 18; bayer += 14, dst += 56 ) { - v_uint16x8 r0 = v_load((ushort*)bayer); - v_uint16x8 r1 = v_load((ushort*)(bayer+bayer_step)); - v_uint16x8 r2 = v_load((ushort*)(bayer+bayer_step*2)); + v_uint16x8 r0 = v_reinterpret_as_u16(v_load(bayer)); + v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step)); + v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2)); v_uint16x8 b1 = (r0 & masklo) + (r2 & masklo); v_uint16x8 nextb1 = v_rotate_right<1>(b1); @@ -494,9 +494,9 @@ public: B G B G | B G B G | B G B G | B G B G */ - v_uint16x8 r0 = v_load((ushort*)bayer); - v_uint16x8 r1 = v_load((ushort*)(bayer+bayer_step)); - v_uint16x8 r2 = v_load((ushort*)(bayer+bayer_step*2)); + v_uint16x8 r0 = v_reinterpret_as_u16(v_load(bayer)); + v_uint16x8 r1 = v_reinterpret_as_u16(v_load(bayer+bayer_step)); + v_uint16x8 r2 = v_reinterpret_as_u16(v_load(bayer+bayer_step*2)); v_uint16x8 b1 = (r0 & masklow) + (r2 & masklow); v_uint16x8 nextb1 = v_rotate_right<1>(b1); From 0f665b3f15573e3779888cf1a2f97a5757a270bf Mon Sep 17 00:00:00 2001 From: Maksim Shabunin Date: Tue, 11 Jul 2023 14:41:32 +0300 Subject: [PATCH 072/105] doc: added environment vars reference --- .../config_reference.markdown | 2 +- .../env_reference/env_reference.markdown | 345 ++++++++++++++++++ .../table_of_content_introduction.markdown | 1 + 3 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 doc/tutorials/introduction/env_reference/env_reference.markdown diff --git a/doc/tutorials/introduction/config_reference/config_reference.markdown b/doc/tutorials/introduction/config_reference/config_reference.markdown index 81607b0086..3ed87e5bdf 100644 --- a/doc/tutorials/introduction/config_reference/config_reference.markdown +++ b/doc/tutorials/introduction/config_reference/config_reference.markdown @@ -2,7 +2,7 @@ OpenCV configuration options reference {#tutorial_config_reference} ====================================== @prev_tutorial{tutorial_general_install} -@next_tutorial{tutorial_linux_install} +@next_tutorial{tutorial_env_reference} @tableofcontents diff --git a/doc/tutorials/introduction/env_reference/env_reference.markdown b/doc/tutorials/introduction/env_reference/env_reference.markdown new file mode 100644 index 0000000000..2a850dc299 --- /dev/null +++ b/doc/tutorials/introduction/env_reference/env_reference.markdown @@ -0,0 +1,345 @@ +OpenCV environment variables reference {#tutorial_env_reference} +====================================== + +@prev_tutorial{tutorial_config_reference} +@next_tutorial{tutorial_linux_install} + +@tableofcontents + +### Introduction + +OpenCV can change its behavior depending on the runtime environment: +- enable extra debugging output or performance tracing +- modify default locations and search paths +- tune some algorithms or general behavior +- enable or disable workarounds, safety features and optimizations + +**Notes:** +- ⭐ marks most popular variables +- variables with names like this `VAR_${NAME}` describes family of variables, where `${NAME}` should be changed to one of predefined values, e.g. `VAR_TBB`, `VAR_OPENMP`, ... + +##### Setting environment variable in Windows +In terminal or cmd-file (bat-file): +```.bat +set MY_ENV_VARIABLE=true +C:\my_app.exe +``` +In GUI: +- Go to "Settings -> System -> About" +- Click on "Advanced system settings" in the right part +- In new window click on the "Environment variables" button +- Add an entry to the "User variables" list + +##### Setting environment variable in Linux + +In terminal or shell script: +```.sh +export MY_ENV_VARIABLE=true +./my_app +``` +or as a single command: +```.sh +MY_ENV_VARIABLE=true ./my_app +``` + +##### Setting environment variable in Python + +```.py +import os +os.environ["MY_ENV_VARIABLE"] = True +import cv2 # variables set after this may not have effect +``` + + +### Types + +- _non-null_ - set to anything to enable feature, in some cases can be interpreted as other types (e.g. path) +- _bool_ - `1`, `True`, `true`, `TRUE` / `0`, `False`, `false`, `FALSE` +- _number_/_size_ - unsigned number, suffixes `MB`, `Mb`, `mb`, `KB`, `Kb`, `kb` +- _string_ - plain string or can have a structure +- _path_ - to file, to directory +- _paths_ - `;`-separated on Windows, `:`-separated on others + + +### General, core +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_SKIP_CPU_BASELINE_CHECK | non-null | | do not check that current CPU supports all features used by the build (baseline) | +| OPENCV_CPU_DISABLE | `,` or `;`-separated | | disable code branches which use CPU features (dispatched code) | +| OPENCV_SETUP_TERMINATE_HANDLER | bool | true (Windows) | use std::set_terminate to install own termination handler | +| OPENCV_LIBVA_RUNTIME | file path | | libva for VA interoperability utils | +| OPENCV_ENABLE_MEMALIGN | bool | true (except static analysis, memory sanitizer, fuzzying, _WIN32?) | enable aligned memory allocations | +| OPENCV_BUFFER_AREA_ALWAYS_SAFE | bool | false | enable safe mode for multi-buffer allocations (each buffer separately) | +| OPENCV_KMEANS_PARALLEL_GRANULARITY | num | 1000 | tune algorithm parallel work distribution parameter `parallel_for_(..., ..., ..., granularity)` | +| OPENCV_DUMP_ERRORS | bool | true (Debug or Android), false (others) | print extra information on exception (log to Android) | +| OPENCV_DUMP_CONFIG | non-null | | print build configuration to stderr (`getBuildInformation`) | +| OPENCV_PYTHON_DEBUG | bool | false | enable extra warnings in Python bindings | +| OPENCV_TEMP_PATH | non-null / path | `/tmp/` (Linux), `/data/local/tmp/` (Android), `GetTempPathA` (Windows) | directory for temporary files | +| OPENCV_DATA_PATH_HINT | paths | | paths for findDataFile | +| OPENCV_DATA_PATH | paths | | paths for findDataFile | +| OPENCV_SAMPLES_DATA_PATH_HINT | paths | | paths for findDataFile | +| OPENCV_SAMPLES_DATA_PATH | paths | | paths for findDataFile | + +Links: +- https://github.com/opencv/opencv/wiki/CPU-optimizations-build-options + + +### Logging +| name | type | default | description | +|------|------|---------|-------------| +| ⭐ OPENCV_LOG_LEVEL | string | | logging level (see accepted values below) | +| OPENCV_LOG_TIMESTAMP | bool | true | logging with timestamps | +| OPENCV_LOG_TIMESTAMP_NS | bool | false | add nsec to logging timestamps | + +##### Levels: +- `0`, `O`, `OFF`, `S`, `SILENT`, `DISABLE`, `DISABLED` +- `F`, `FATAL` +- `E`, `ERROR` +- `W`, `WARNING`, `WARN`, `WARNINGS` +- `I`, `INFO` +- `D`, `DEBUG` +- `V`, `VERBOSE` + + +### core/parallel_for +| name | type | default | description | +|------|------|---------|-------------| +| ⭐ OPENCV_FOR_THREADS_NUM | num | 0 | set number of threads | +| OPENCV_THREAD_POOL_ACTIVE_WAIT_PAUSE_LIMIT | num | 16 | tune pthreads parallel_for backend | +| OPENCV_THREAD_POOL_ACTIVE_WAIT_WORKER | num | 2000 | tune pthreads parallel_for backend | +| OPENCV_THREAD_POOL_ACTIVE_WAIT_MAIN | num | 10000 | tune pthreads parallel_for backend | +| OPENCV_THREAD_POOL_ACTIVE_WAIT_THREADS_LIMIT | num | 0 | tune pthreads parallel_for backend | +| OPENCV_FOR_OPENMP_DYNAMIC_DISABLE | bool | false | use single OpenMP thread | + + +### backends +OPENCV_LEGACY_WAITKEY +Some modules have multiple available backends, following variables allow choosing specific backend or changing default priorities in which backends will be probed (e.g. when opening a video file). + +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_PARALLEL_BACKEND | string | | choose specific paralel_for backend (one of `TBB`, `ONETBB`, `OPENMP`) | +| OPENCV_PARALLEL_PRIORITY_${NAME} | num | | set backend priority, default is 1000 | +| OPENCV_PARALLEL_PRIORITY_LIST | string, `,`-separated | | list of backends in priority order | +| OPENCV_UI_BACKEND | string | | choose highgui backend for window rendering (one of `GTK`, `GTK3`, `GTK2`, `QT`, `WIN32`) | +| OPENCV_UI_PRIORITY_${NAME} | num | | set highgui backend priority, default is 1000 | +| OPENCV_UI_PRIORITY_LIST | string, `,`-separated | | list of hioghgui backends in priority order | +| OPENCV_VIDEOIO_PRIORITY_${NAME} | num | | set videoio backend priority, default is 1000 | +| OPENCV_VIDEOIO_PRIORITY_LIST | string, `,`-separated | | list of videoio backends in priority order | + + +### plugins +Some external dependencies can be detached into a dynamic library, which will be loaded at runtime (plugin). Following variables allow changing default search locations and naming pattern for these plugins. +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_CORE_PLUGIN_PATH | paths | | directories to search for _core_ plugins | +| OPENCV_CORE_PARALLEL_PLUGIN_${NAME} | string, glob | | parallel_for plugin library name (glob), e.g. default for TBB is "opencv_core_parallel_tbb*.so" | +| OPENCV_DNN_PLUGIN_PATH | paths | | directories to search for _dnn_ plugins | +| OPENCV_DNN_PLUGIN_${NAME} | string, glob | | parallel_for plugin library name (glob), e.g. default for TBB is "opencv_core_parallel_tbb*.so" | +| OPENCV_CORE_PLUGIN_PATH | paths | | directories to search for _highgui_ plugins (YES it is CORE) | +| OPENCV_UI_PLUGIN_${NAME} | string, glob | | _highgui_ plugin library name (glob) | +| OPENCV_VIDEOIO_PLUGIN_PATH | paths | | directories to search for _videoio_ plugins | +| OPENCV_VIDEOIO_PLUGIN_${NAME} | string, glob | | _videoio_ plugin library name (glob) | + +### OpenCL + +**Note:** OpenCL device specification format is `::`, e.g. `AMD:GPU:` + +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_OPENCL_RUNTIME | filepath or `disabled` | | path to OpenCL runtime library (e.g. `OpenCL.dll`, `libOpenCL.so`) | +| ⭐ OPENCV_OPENCL_DEVICE | string or `disabled` | | choose specific OpenCL device. See specification format in the note above. See more details in the Links section. | +| OPENCV_OPENCL_RAISE_ERROR | bool | false | raise exception if something fails during OpenCL kernel preparation and execution (Release builds only) | +| OPENCV_OPENCL_ABORT_ON_BUILD_ERROR | bool | false | abort if OpenCL kernel compilation failed | +| OPENCV_OPENCL_CACHE_ENABLE | bool | true | enable OpenCL kernel cache | +| OPENCV_OPENCL_CACHE_WRITE | bool | true | allow writing to the cache, otherwise cache will be read-only | +| OPENCV_OPENCL_CACHE_LOCK_ENABLE | bool | true | use .lock files to synchronize between multiple applications using the same OpenCL cache (may not work on network drives) | +| OPENCV_OPENCL_CACHE_CLEANUP | bool | true | automatically remove old entries from cache (leftovers from older OpenCL runtimes) | +| OPENCV_OPENCL_VALIDATE_BINARY_PROGRAMS | bool | false | validate loaded binary OpenCL kernels | +| OPENCV_OPENCL_DISABLE_BUFFER_RECT_OPERATIONS | bool | true (Apple), false (others) | enable workaround for non-continuos data downloads | +| OPENCV_OPENCL_BUILD_EXTRA_OPTIONS | string | | pass extra options to OpenCL kernel compilation | +| OPENCV_OPENCL_ENABLE_MEM_USE_HOST_PTR | bool | true | workaround/optimization for buffer allocation | +| OPENCV_OPENCL_ALIGNMENT_MEM_USE_HOST_PTR | num | 4 | parameter for OPENCV_OPENCL_ENABLE_MEM_USE_HOST_PTR | +| OPENCV_OPENCL_DEVICE_MAX_WORK_GROUP_SIZE | num | 0 | allow to decrease maxWorkGroupSize | +| OPENCV_OPENCL_PROGRAM_CACHE | num | 0 | limit number of programs in OpenCL kernel cache | +| OPENCV_OPENCL_RAISE_ERROR_REUSE_ASYNC_KERNEL | bool | false | raise exception if async kernel failed | +| OPENCV_OPENCL_BUFFERPOOL_LIMIT | num | 1 << 27 (Intel device), 0 (others) | limit memory used by buffer bool | +| OPENCV_OPENCL_HOST_PTR_BUFFERPOOL_LIMIT | num | | same as OPENCV_OPENCL_BUFFERPOOL_LIMIT, but for HOST_PTR buffers | +| OPENCV_OPENCL_BUFFER_FORCE_MAPPING | bool | false | force clEnqueueMapBuffer | +| OPENCV_OPENCL_BUFFER_FORCE_COPYING | bool | false | force clEnqueueReadBuffer/clEnqueueWriteBuffer | +| OPENCV_OPENCL_FORCE | bool | false | force running OpenCL kernel even if usual conditions are not met (e.g. dst.isUMat) | +| OPENCV_OPENCL_PERF_CHECK_BYPASS | bool | false | force running OpenCL kernel even if usual performance-related conditions are not met (e.g. image is very small) | + +##### SVM (Shared Virtual Memory) - disabled by default +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_OPENCL_SVM_DISABLE | bool | false | disable SVM | +| OPENCV_OPENCL_SVM_FORCE_UMAT_USAGE | bool | false | | +| OPENCV_OPENCL_SVM_DISABLE_UMAT_USAGE | bool | false | | +| OPENCV_OPENCL_SVM_CAPABILITIES_MASK | num | | | +| OPENCV_OPENCL_SVM_BUFFERPOOL_LIMIT | num | | same as OPENCV_OPENCL_BUFFERPOOL_LIMIT, but for SVM buffers | + +##### Links: +- https://github.com/opencv/opencv/wiki/OpenCL-optimizations + + +### Tracing/Profiling +| name | type | default | description | +|------|------|---------|-------------| +| ⭐ OPENCV_TRACE | bool | false | enable trace | +| OPENCV_TRACE_LOCATION | string | `OpenCVTrace` | trace file name ("${name}-$03d.txt") | +| OPENCV_TRACE_DEPTH_OPENCV | num | 1 | | +| OPENCV_TRACE_MAX_CHILDREN_OPENCV | num | 1000 | | +| OPENCV_TRACE_MAX_CHILDREN | num | 1000 | | +| OPENCV_TRACE_SYNC_OPENCL | bool | false | wait for OpenCL kernels to finish | +| OPENCV_TRACE_ITT_ENABLE | bool | true | | +| OPENCV_TRACE_ITT_PARENT | bool | false | set parentID for ITT task | +| OPENCV_TRACE_ITT_SET_THREAD_NAME | bool | false | set name for OpenCV's threads "OpenCVThread-%03d" | + +##### Links: +- https://github.com/opencv/opencv/wiki/Profiling-OpenCV-Applications + + +##### Cache +**Note:** Default tmp location is `%TMPDIR%` (Windows); `$XDG_CACHE_HOME`, `$HOME/.cache`, `/var/tmp`, `/tmp` (others) +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_CACHE_SHOW_CLEANUP_MESSAGE | bool | true | show cache cleanup message | +| OPENCV_DOWNLOAD_CACHE_DIR | path | default tmp location | cache directory for downloaded files (subdirectory `downloads`) | +| OPENCV_DNN_IE_GPU_CACHE_DIR | path | default tmp location | cache directory for OpenVINO OpenCL kernels (subdirectory `dnn_ie_cache_${device}`) | +| OPENCV_OPENCL_CACHE_DIR | path | default tmp location | cache directory for OpenCL kernels cache (subdirectory `opencl_cache`) | + + +### dnn +**Note:** In the table below `dump_base_name` equals to `ocv_dnn_net_%05d_%02d` where first argument is internal network ID and the second - dump level. +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_DNN_BACKEND_DEFAULT | num | 3 (OpenCV) | set default DNN backend, see dnn.hpp for backends enumeration | +| OPENCV_DNN_NETWORK_DUMP | num | 0 | level of information dumps, 0 - no dumps (default file name `${dump_base_name}.dot`) | +| OPENCV_DNN_DISABLE_MEMORY_OPTIMIZATIONS | bool | false | | +| OPENCV_DNN_CHECK_NAN_INF | bool | false | check for NaNs in layer outputs | +| OPENCV_DNN_CHECK_NAN_INF_DUMP | bool | false | print layer data when NaN check has failed | +| OPENCV_DNN_CHECK_NAN_INF_RAISE_ERROR | bool | false | also raise exception when NaN check has failed | +| OPENCV_DNN_ONNX_USE_LEGACY_NAMES | bool | false | use ONNX node names as-is instead of "onnx_node!${node_name}" | +| OPENCV_DNN_CUSTOM_ONNX_TYPE_INCLUDE_DOMAIN_NAME | bool | true | prepend layer domain to layer types ("domain.type") | +| OPENCV_VULKAN_RUNTIME | file path | | set location of Vulkan runtime library for DNN Vulkan backend | +| OPENCV_DNN_IE_SERIALIZE | bool | false | dump intermediate OpenVINO graph (default file names `${dump_base_name}_ngraph.xml`, `${dump_base_name}_ngraph.bin`) | +| OPENCV_DNN_IE_EXTRA_PLUGIN_PATH | path | | path to extra OpenVINO plugins | +| OPENCV_DNN_IE_VPU_TYPE | string | | Force using specific OpenVINO VPU device type ("Myriad2" or "MyriadX") | +| OPENCV_TEST_DNN_IE_VPU_TYPE | string | | same as OPENCV_DNN_IE_VPU_TYPE, but for tests | +| OPENCV_DNN_INFERENCE_ENGINE_HOLD_PLUGINS | bool | true | always hold one existing OpenVINO instance to avoid crashes on unloading | +| OPENCV_DNN_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND | bool | true (Windows), false (other) | another OpenVINO lifetime workaround | +| OPENCV_DNN_OPENCL_ALLOW_ALL_DEVICES | bool | false | allow running on CPU devices, allow FP16 on non-Intel device | +| OPENCV_OCL4DNN_CONVOLUTION_IGNORE_INPUT_DIMS_4_CHECK | bool | false | workaround for OpenCL backend, see https://github.com/opencv/opencv/issues/20833 | +| OPENCV_OCL4DNN_WORKAROUND_IDLF | bool | true | another workaround for OpenCL backend | +| OPENCV_OCL4DNN_CONFIG_PATH | path | | path to kernel configuration cache for auto-tuning (must be existing directory), set this variable to enable auto-tuning | +| OPENCV_OCL4DNN_DISABLE_AUTO_TUNING | bool | false | disable auto-tuning | +| OPENCV_OCL4DNN_FORCE_AUTO_TUNING | bool | false | force auto-tuning | +| OPENCV_OCL4DNN_TEST_ALL_KERNELS | num | 0 | test convolution kernels, number of iterations (auto-tuning) | +| OPENCV_OCL4DNN_DUMP_FAILED_RESULT | bool | false | dump extra information on errors (auto-tuning) | +| OPENCV_OCL4DNN_TUNING_RAISE_CHECK_ERROR | bool | false | raise exception on errors (auto-tuning) | + + +### Tests +| name | type | default | description | +|------|------|---------|-------------| +| ⭐ OPENCV_TEST_DATA_PATH | dir path | | set test data search location (e.g. `/home/user/opencv_extra/testdata`) | +| ⭐ OPENCV_DNN_TEST_DATA_PATH | dir path | `$OPENCV_TEST_DATA_PATH/dnn` | set DNN model search location for tests (used by _dnn_, _gapi_, _objdetect_, _video_ modules) | +| OPENCV_OPEN_MODEL_ZOO_DATA_PATH | dir path | `$OPENCV_DNN_TEST_DATA_PATH/omz_intel_models` | set OpenVINO models search location for tests (used by _dnn_, _gapi_ modules) | +| INTEL_CVSDK_DIR | | | some _dnn_ tests can search OpenVINO models here too | +| OPENCV_TEST_DEBUG | num | 0 | debug level for tests, same as `--test_debug` (0 - no debug (default), 1 - basic test debug information, >1 - extra debug information) | +| OPENCV_TEST_REQUIRE_DATA | bool | false | same as `--test_require_data` option (fail on missing non-required test data instead of skip) | +| OPENCV_TEST_CHECK_OPTIONAL_DATA | bool | false | assert when optional data is not found | +| OPENCV_IPP_CHECK | bool | false | default value for `--test_ipp_check` and `--perf_ipp_check` | +| OPENCV_PERF_VALIDATION_DIR | dir path | | location of files read/written by `--perf_read_validation_results`/`--perf_write_validation_results` | +| ⭐ OPENCV_PYTEST_FILTER | string (glob) | | test filter for Python tests | + +##### Links: +* https://github.com/opencv/opencv/wiki/QA_in_OpenCV + + +### videoio +**Note:** extra FFmpeg options should be pased in form `key;value|key;value|key;value`, for example `hwaccel;cuvid|video_codec;h264_cuvid|vsync;0` or `vcodec;x264|vprofile;high|vlevel;4.0` + +| name | type | default | description | +|------|------|---------|-------------| +| ⭐ OPENCV_FFMPEG_CAPTURE_OPTIONS | string (see note) | | extra options for VideoCapture FFmpeg backend | +| ⭐ OPENCV_FFMPEG_WRITER_OPTIONS | string (see note) | | extra options for VideoWriter FFmpeg backend | +| OPENCV_FFMPEG_THREADS | num | | set FFmpeg thread count | +| OPENCV_FFMPEG_DEBUG | non-null | | enable logging messages from FFmpeg | +| OPENCV_FFMPEG_LOGLEVEL | num | | set FFmpeg logging level | +| OPENCV_FFMPEG_DLL_DIR | dir path | | directory with FFmpeg plugin (legacy) | +| OPENCV_FFMPEG_IS_THREAD_SAFE | bool | false | enabling this option will turn off thread safety locks in the FFmpeg backend (use only if you are sure FFmpeg is built with threading support, tested on Linux) | +| OPENCV_FFMPEG_READ_ATTEMPTS | num | 4096 | number of failed `av_read_frame` attempts before failing read procedure | +| OPENCV_FFMPEG_DECODE_ATTEMPTS | num | 64 | number of failed `avcodec_receive_frame` attempts before failing decoding procedure | +| OPENCV_VIDEOIO_GSTREAMER_CALL_DEINIT | bool | false | close GStreamer instance on end | +| OPENCV_VIDEOIO_GSTREAMER_START_MAINLOOP | bool | false | start GStreamer loop in separate thread | +| OPENCV_VIDEOIO_MFX_IMPL | num | | set specific MFX implementation (see MFX docs for enumeration) | +| OPENCV_VIDEOIO_MFX_EXTRA_SURFACE_NUM | num | 1 | add extra surfaces to the surface pool | +| OPENCV_VIDEOIO_MFX_POOL_TIMEOUT | num | 1 | timeout for waiting for free surface from the pool (in seconds) | +| OPENCV_VIDEOIO_MFX_BITRATE_DIVISOR | num | 300 | this option allows to tune encoding bitrate (video quality/size) | +| OPENCV_VIDEOIO_MFX_WRITER_TIMEOUT | num | 1 | timeout for encoding operation (in seconds) | +| OPENCV_VIDEOIO_MSMF_ENABLE_HW_TRANSFORMS | bool | true | allow HW-accelerated transformations (DXVA) in MediaFoundation processing graph (may slow down camera probing process) | +| OPENCV_DSHOW_DEBUG | non-null | | enable verbose logging in the DShow backend | +| OPENCV_DSHOW_SAVEGRAPH_FILENAME | file path | | enable processing graph tump in the DShow backend | +| OPENCV_VIDEOIO_V4L_RANGE_NORMALIZED | bool | false | use (0, 1) range for properties (V4L) | +| OPENCV_VIDEOIO_V4L_SELECT_TIMEOUT | num | 10 | timeout for select call (in seconds) (V4L) | +| OPENCV_VIDEOCAPTURE_DEBUG | bool | false | enable debug messages for VideoCapture | +| OPENCV_VIDEOWRITER_DEBUG | bool | false | enable debug messages for VideoWriter | +| ⭐ OPENCV_VIDEOIO_DEBUG | bool | false | debug messages for both VideoCapture and VideoWriter | + +##### videoio tests +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_TEST_VIDEOIO_BACKEND_REQUIRE_FFMPEG | | | test app will exit if no FFmpeg backend is available | +| OPENCV_TEST_V4L2_VIVID_DEVICE | file path | | path to VIVID virtual camera device for V4L2 test (e.g. `/dev/video5`) | +| OPENCV_TEST_PERF_CAMERA_LIST | paths | | cameras to use in performance test (waitAny_V4L test) | +| OPENCV_TEST_CAMERA_%d_FPS | num | | fps to set for N-th camera (0-based index) (waitAny_V4L test) | + + +### gapi +| name | type | default | description | +|------|------|---------|-------------| +| ⭐ GRAPH_DUMP_PATH | file path | | dump graph (dot format) | +| PIPELINE_MODELS_PATH | dir path | | pipeline_modeling_tool sample application uses this var | +| OPENCV_GAPI_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND | bool | true (Windows, Apple), false (others) | similar to OPENCV_DNN_INFERENCE_ENGINE_CORE_LIFETIME_WORKAROUND | + +##### gapi tests/samples +| name | type | default | description | +|------|------|---------|-------------| +| PLAIDML_DEVICE | string | | specific to PlaidML backend test | +| PLAIDML_TARGET | string | | specific to PlaidML backend test | +| OPENCV_GAPI_ONNX_MODEL_PATH | dir path | | search location for ONNX models test | +| OPENCV_TEST_FREETYPE_FONT_PATH | file path | | location of TrueType font for one of tests | + +##### Links: +* https://github.com/opencv/opencv/wiki/Using-G-API-with-OpenVINO-Toolkit +* https://github.com/opencv/opencv/wiki/Using-G-API-with-MS-ONNX-Runtime + + +### highgui + +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_LEGACY_WAITKEY | non-null | | switch `waitKey` return result (default behavior: `return code & 0xff` (or -1), legacy behavior: `return code`) | +| $XDG_RUNTIME_DIR | | | Wayland backend specific - create shared memory-mapped file for interprocess communication (named `opencv-shared-??????`) | + + +### imgproc +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_OPENCL_IMGPROC_MORPH_SPECIAL_KERNEL | bool | true (Apple), false (others) | use special OpenCL kernel for small morph kernel (Intel devices) | +| OPENCV_GAUSSIANBLUR_CHECK_BITEXACT_KERNELS | bool | false | validate Gaussian kernels before running (src is CV_16U, bit-exact version) | + + +### imgcodecs +| name | type | default | description | +|------|------|---------|-------------| +| OPENCV_IMGCODECS_AVIF_MAX_FILE_SIZE | num | 64MB | limit input AVIF size | +| OPENCV_IMGCODECS_WEBP_MAX_FILE_SIZE | num | 64MB | limit input WEBM size | +| OPENCV_IO_MAX_IMAGE_PARAMS | num | 50 | limit maximum allowed number of parameters in imwrite and imencode | +| OPENCV_IO_MAX_IMAGE_WIDTH | num | 1 << 20, limit input image size to avoid large memory allocations | | +| OPENCV_IO_MAX_IMAGE_HEIGHT | num | 1 << 20 | | +| OPENCV_IO_MAX_IMAGE_PIXELS | num | 1 << 30 | | +| OPENCV_IO_ENABLE_OPENEXR | bool | true (set build option OPENCV_IO_FORCE_OPENEXR or use external OpenEXR), false (otherwise) | enable OpenEXR backend | +| OPENCV_IO_ENABLE_JASPER | bool | true (set build option OPENCV_IO_FORCE_JASPER), false (otherwise) | enable Jasper backend | diff --git a/doc/tutorials/introduction/table_of_content_introduction.markdown b/doc/tutorials/introduction/table_of_content_introduction.markdown index 8fa89d7d7f..4df7ad2c78 100644 --- a/doc/tutorials/introduction/table_of_content_introduction.markdown +++ b/doc/tutorials/introduction/table_of_content_introduction.markdown @@ -3,6 +3,7 @@ Introduction to OpenCV {#tutorial_table_of_content_introduction} - @subpage tutorial_general_install - @subpage tutorial_config_reference +- @subpage tutorial_env_reference ##### Linux - @subpage tutorial_linux_install From fdfb87520815cbfce32ce587b9c198166ea106e0 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Wed, 12 Jul 2023 14:20:01 +0200 Subject: [PATCH 073/105] Merge pull request #23922 from vrabaud:imgwarp Fix imgwarp at borders when transparent. #23922 I believe this is a proper fix to #23562 The PR #23754 overwrites data while that should not be the case with transparent data. The original test is failing because points at the border do not get computed because they do not have 4 neighbors to be computed. Still ,we can approximate their computation with whatever neighbors that are available. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/imgproc/src/imgwarp.cpp | 48 +++++++++++++++++---------- modules/imgproc/test/test_imgwarp.cpp | 22 ++++++++++++ 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/modules/imgproc/src/imgwarp.cpp b/modules/imgproc/src/imgwarp.cpp index 268555bada..e5d9b0defb 100644 --- a/modules/imgproc/src/imgwarp.cpp +++ b/modules/imgproc/src/imgwarp.cpp @@ -757,6 +757,37 @@ static void remapBilinear( const Mat& _src, Mat& _dst, const Mat& _xy, } else { + if (borderType == BORDER_TRANSPARENT) { + for (; dx < X1; dx++, D += cn) { + if (dx >= dsize.width) continue; + const int sx = XY[dx * 2], sy = XY[dx * 2 + 1]; + // If the mapped point is still within bounds, it did not get computed + // because it lacked 4 neighbors. Still, it can be computed with an + // approximate formula. If it is outside, the point is left untouched. + if (sx >= 0 && sx <= ssize.width - 1 && sy >= 0 && sy <= ssize.height - 1) { + const AT* w = wtab + FXY[dx] * 4; + WT w_tot = 0; + if (sx >= 0 && sy >= 0) w_tot += w[0]; + if (sy >= 0 && sx < ssize.width - 1) w_tot += w[1]; + if (sx >= 0 && sy < ssize.height - 1) w_tot += w[2]; + if (sx < ssize.width - 1 && sy < ssize.height - 1) w_tot += w[3]; + if (w_tot == 0.f) continue; + const WT w_tot_ini = (WT)w[0] + w[1] + w[2] + w[3]; + const T* S = S0 + sy * sstep + sx * cn; + for (int k = 0; k < cn; k++) { + WT t0 = 0; + if (sx >= 0 && sy >= 0) t0 += S[k] * w[0]; + if (sy >= 0 && sx < ssize.width - 1) t0 += S[k + cn] * w[1]; + if (sx >= 0 && sy < ssize.height - 1) t0 += S[sstep + k] * w[2]; + if (sx < ssize.width - 1 && sy < ssize.height - 1) t0 += S[sstep + k + cn] * w[3]; + t0 = (WT)(t0 * (float)w_tot_ini / w_tot); + D[k] = castOp(t0); + } + } + } + continue; + } + if( cn == 1 ) for( ; dx < X1; dx++, D++ ) { @@ -767,12 +798,6 @@ static void remapBilinear( const Mat& _src, Mat& _dst, const Mat& _xy, { D[0] = cval[0]; } - else if (borderType == BORDER_TRANSPARENT) - { - if (sx < ssize.width && sx >= 0 && - sy < ssize.height && sy >= 0) - D[0] = S0[sy*sstep + sx]; - } else { int sx0, sx1, sy0, sy1; @@ -814,13 +839,6 @@ static void remapBilinear( const Mat& _src, Mat& _dst, const Mat& _xy, for(int k = 0; k < cn; k++ ) D[k] = cval[k]; } - else if (borderType == BORDER_TRANSPARENT) - { - if (sx < ssize.width && sx >= 0 && - sy < ssize.height && sy >= 0) - for(int k = 0; k < cn; k++ ) - D[k] = S0[sy*sstep + sx*cn + k]; - } else { int sx0, sx1, sy0, sy1; @@ -837,10 +855,6 @@ static void remapBilinear( const Mat& _src, Mat& _dst, const Mat& _xy, v2 = S0 + sy1*sstep + sx0*cn; v3 = S0 + sy1*sstep + sx1*cn; } - else if( borderType == BORDER_TRANSPARENT && - ((unsigned)sx >= (unsigned)(ssize.width-1) || - (unsigned)sy >= (unsigned)(ssize.height-1))) - continue; else { sx0 = borderInterpolate(sx, ssize.width, borderType); diff --git a/modules/imgproc/test/test_imgwarp.cpp b/modules/imgproc/test/test_imgwarp.cpp index 68208caa04..4ba2ab80b4 100644 --- a/modules/imgproc/test/test_imgwarp.cpp +++ b/modules/imgproc/test/test_imgwarp.cpp @@ -1672,6 +1672,28 @@ TEST(Imgproc_Remap, issue_23562) remap(src, dst, mapx, mapy, INTER_LINEAR, BORDER_TRANSPARENT); ASSERT_EQ(0.0, cvtest::norm(ref, dst, NORM_INF)) << "channels=" << cn; } + + mapx = Mat1f({3, 3}, {0, 1, 2, 0, 1, 2, 0, 1, 2}); + mapy = Mat1f({3, 3}, {0, 0, 0, 1, 1, 1, 2, 2, 1.5}); + for (int cn = 1; cn <= 4; ++cn) { + Mat src = cv::Mat(3, 3, CV_32FC(cn)); + Mat dst = 10 * Mat::ones(3, 3, CV_32FC(cn)); + for(int y = 0; y < 3; ++y) { + for(int x = 0; x < 3; ++x) { + for(int k = 0; k < cn; ++k) { + src.ptr(y,x)[k] = 10.f * y + x; + } + } + } + + Mat ref = src.clone(); + for(int k = 0; k < cn; ++k) { + ref.ptr(2,2)[k] = (src.ptr(1, 2)[k] + src.ptr(2, 2)[k]) / 2.f; + } + + remap(src, dst, mapx, mapy, INTER_LINEAR, BORDER_TRANSPARENT); + ASSERT_EQ(0.0, cvtest::norm(ref, dst, NORM_INF)) << "channels=" << cn; + } } }} // namespace From 0fef0f2ae09b27cfa1111b292532c1f72121c46f Mon Sep 17 00:00:00 2001 From: headshog Date: Thu, 13 Jul 2023 13:21:21 +0300 Subject: [PATCH 074/105] fix numtrunc at tiff_dirread.c --- 3rdparty/libtiff/tif_dirread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/libtiff/tif_dirread.c b/3rdparty/libtiff/tif_dirread.c index ba127ca917..45c1107697 100644 --- a/3rdparty/libtiff/tif_dirread.c +++ b/3rdparty/libtiff/tif_dirread.c @@ -4371,7 +4371,7 @@ static void TIFFReadDirectoryCheckOrder(TIFF* tif, TIFFDirEntry* dir, uint16 dircount) { static const char module[] = "TIFFReadDirectoryCheckOrder"; - uint16 m; + uint32 m; uint16 n; TIFFDirEntry* o; m=0; From c12e1ecb866bf64d23b8a054ba01afefa410762d Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 4 Jul 2023 02:44:30 +0300 Subject: [PATCH 075/105] update aruco bytesList docs --- doc/opencv.bib | 8 ++++ .../opencv2/objdetect/aruco_dictionary.hpp | 39 +++++++++++-------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/doc/opencv.bib b/doc/opencv.bib index 64aa363202..6071cc3e1b 100644 --- a/doc/opencv.bib +++ b/doc/opencv.bib @@ -1377,3 +1377,11 @@ year={2005}, pages={70-74} } +@inproceedings{wang2016iros, + AUTHOR = {John Wang and Edwin Olson}, + TITLE = {{AprilTag} 2: Efficient and robust fiducial detection}, + BOOKTITLE = {Proceedings of the {IEEE/RSJ} International Conference on Intelligent + Robots and Systems {(IROS)}}, + YEAR = {2016}, + MONTH = {October}, +} diff --git a/modules/objdetect/include/opencv2/objdetect/aruco_dictionary.hpp b/modules/objdetect/include/opencv2/objdetect/aruco_dictionary.hpp index c46b5fbfb5..bc7b934b2a 100644 --- a/modules/objdetect/include/opencv2/objdetect/aruco_dictionary.hpp +++ b/modules/objdetect/include/opencv2/objdetect/aruco_dictionary.hpp @@ -13,32 +13,39 @@ namespace aruco { //! @{ -/** @brief Dictionary/Set of markers, it contains the inner codification +/** @brief Dictionary is a set of unique ArUco markers of the same size * - * BytesList contains the marker codewords where: + * `bytesList` storing as 2-dimensions Mat with 4-th channels (CV_8UC4 type was used) and contains the marker codewords where: * - bytesList.rows is the dictionary size - * - each marker is encoded using `nbytes = ceil(markerSize*markerSize/8.)` + * - each marker is encoded using `nbytes = ceil(markerSize*markerSize/8.)` bytes * - each row contains all 4 rotations of the marker, so its length is `4*nbytes` - * - * `bytesList.ptr(i)[k*nbytes + j]` is then the j-th byte of i-th marker, in its k-th rotation. + * - the byte order in the bytesList[i] row: + * `//bytes without rotation/bytes with rotation 1/bytes with rotation 2/bytes with rotation 3//` + * So `bytesList.ptr(i)[k*nbytes + j]` is the j-th byte of i-th marker, in its k-th rotation. + * @note Python bindings generate matrix with shape of bytesList `dictionary_size x nbytes x 4`, + * but it should be indexed like C++ version. Python example for j-th byte of i-th marker, in its k-th rotation: + * `aruco_dict.bytesList[id].ravel()[k*nbytes + j]` */ class CV_EXPORTS_W_SIMPLE Dictionary { public: - CV_PROP_RW Mat bytesList; // marker code information - CV_PROP_RW int markerSize; // number of bits per dimension - CV_PROP_RW int maxCorrectionBits; // maximum number of bits that can be corrected - + CV_PROP_RW Mat bytesList; ///< marker code information. See class description for more details + CV_PROP_RW int markerSize; ///< number of bits per dimension + CV_PROP_RW int maxCorrectionBits; ///< maximum number of bits that can be corrected CV_WRAP Dictionary(); + /** @brief Basic ArUco dictionary constructor + * + * @param bytesList bits for all ArUco markers in dictionary see memory layout in the class description + * @param _markerSize ArUco marker size in units + * @param maxcorr maximum number of bits that can be corrected + */ CV_WRAP Dictionary(const Mat &bytesList, int _markerSize, int maxcorr = 0); - - /** @brief Read a new dictionary from FileNode. * - * Dictionary format:\n + * Dictionary example in YAML format:\n * nmarkers: 35\n * markersize: 6\n * maxCorrectionBits: 5\n @@ -54,13 +61,13 @@ class CV_EXPORTS_W_SIMPLE Dictionary { /** @brief Given a matrix of bits. Returns whether if marker is identified or not. * - * It returns by reference the correct id (if any) and the correct rotation + * Returns reference to the marker id in the dictionary (if any) and its rotation. */ CV_WRAP bool identify(const Mat &onlyBits, CV_OUT int &idx, CV_OUT int &rotation, double maxCorrectionRate) const; - /** @brief Returns the distance of the input bits to the specific id. + /** @brief Returns Hamming distance of the input bits to the specific id. * - * If allRotations is true, the four posible bits rotation are considered + * If `allRotations` flag is set, the four posible marker rotations are considered */ CV_WRAP int getDistanceToId(InputArray bits, int id, bool allRotations = true) const; @@ -70,7 +77,7 @@ class CV_EXPORTS_W_SIMPLE Dictionary { CV_WRAP void generateImageMarker(int id, int sidePixels, OutputArray _img, int borderBits = 1) const; - /** @brief Transform matrix of bits to list of bytes in the 4 rotations + /** @brief Transform matrix of bits to list of bytes with 4 marker rotations */ CV_WRAP static Mat getByteListFromBits(const Mat &bits); From 52d9685cb99f2ae4393887bec85a07f81c780128 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Wed, 5 Jul 2023 14:09:20 +0300 Subject: [PATCH 076/105] Fixed possible out-of-bound access in circles drawing. --- modules/imgproc/src/drawing.cpp | 10 +++++----- modules/imgproc/test/test_drawing.cpp | 8 ++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) mode change 100755 => 100644 modules/imgproc/src/drawing.cpp diff --git a/modules/imgproc/src/drawing.cpp b/modules/imgproc/src/drawing.cpp old mode 100755 new mode 100644 index a0cd8ae772..cbd238aca9 --- a/modules/imgproc/src/drawing.cpp +++ b/modules/imgproc/src/drawing.cpp @@ -1556,7 +1556,7 @@ Circle( Mat& img, Point center, int radius, const void* color, int fill ) ICV_HLINE( tptr1, x21, x22, color, pix_size ); } } - else if( x11 < size.width && x12 >= 0 && y21 < size.height && y22 >= 0 ) + else if( x11 < size.width && x12 >= 0 && y21 < size.height && y22 >= 0) { if( fill ) { @@ -1564,7 +1564,7 @@ Circle( Mat& img, Point center, int radius, const void* color, int fill ) x12 = MIN( x12, size.width - 1 ); } - if( (unsigned)y11 < (unsigned)size.height ) + if( y11 >= 0 && y11 < size.height ) { uchar *tptr = ptr + y11 * step; @@ -1579,7 +1579,7 @@ Circle( Mat& img, Point center, int radius, const void* color, int fill ) ICV_HLINE( tptr, x11, x12, color, pix_size ); } - if( (unsigned)y12 < (unsigned)size.height ) + if( y12 >= 0 && y12 < size.height ) { uchar *tptr = ptr + y12 * step; @@ -1602,7 +1602,7 @@ Circle( Mat& img, Point center, int radius, const void* color, int fill ) x22 = MIN( x22, size.width - 1 ); } - if( (unsigned)y21 < (unsigned)size.height ) + if( y21 >= 0 && y21 < size.height ) { uchar *tptr = ptr + y21 * step; @@ -1617,7 +1617,7 @@ Circle( Mat& img, Point center, int radius, const void* color, int fill ) ICV_HLINE( tptr, x21, x22, color, pix_size ); } - if( (unsigned)y22 < (unsigned)size.height ) + if( y22 >= 0 && y22 < size.height ) { uchar *tptr = ptr + y22 * step; diff --git a/modules/imgproc/test/test_drawing.cpp b/modules/imgproc/test/test_drawing.cpp index 8afd0e0072..23208ab881 100644 --- a/modules/imgproc/test/test_drawing.cpp +++ b/modules/imgproc/test/test_drawing.cpp @@ -913,4 +913,12 @@ INSTANTIATE_TEST_CASE_P( ) ); +TEST(Drawing, circle_overflow) +{ + applyTestTag(CV_TEST_TAG_VERYLONG); + cv::Mat1b matrix = cv::Mat1b::zeros(600, 600); + cv::Scalar kBlue = cv::Scalar(0, 0, 255); + cv::circle(matrix, cv::Point(275, -2147483318), 2147483647, kBlue, 1, 8, 0); +} + }} // namespace From 7819ec784bff61271184146cb1518592005db80b Mon Sep 17 00:00:00 2001 From: firebladed <34522909+firebladed@users.noreply.github.com> Date: Fri, 14 Jul 2023 09:31:55 +0100 Subject: [PATCH 077/105] Merge pull request #18498 from firebladed:patch-1 Add V4L2_PIX_FMT_Y16_BE pixel format #18498 Address #18495 relates to #23944 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or other license that is incompatible with OpenCV - [ ] The PR is proposed to proper branch - [x] There is reference to original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake - [ ] Test using Melexis MLX90640 --- modules/videoio/src/cap_v4l.cpp | 26 +++++++++++++++++++++++++- modules/videoio/test/test_precomp.hpp | 3 ++- modules/videoio/test/test_v4l2.cpp | 8 ++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/modules/videoio/src/cap_v4l.cpp b/modules/videoio/src/cap_v4l.cpp index 2f8b8b3745..e3c53d7cdd 100644 --- a/modules/videoio/src/cap_v4l.cpp +++ b/modules/videoio/src/cap_v4l.cpp @@ -268,6 +268,14 @@ typedef uint32_t __u32; #define V4L2_PIX_FMT_Y12 v4l2_fourcc('Y', '1', '2', ' ') #endif +#ifndef V4L2_PIX_FMT_Y16 +#define V4L2_PIX_FMT_Y16 v4l2_fourcc('Y', '1', '6', ' ') +#endif + +#ifndef V4L2_PIX_FMT_Y16_BE +#define V4L2_PIX_FMT_Y16_BE v4l2_fourcc_be('Y', '1', '6', ' ') +#endif + #ifndef V4L2_PIX_FMT_ABGR32 #define V4L2_PIX_FMT_ABGR32 v4l2_fourcc('A', 'R', '2', '4') #endif @@ -609,6 +617,7 @@ bool CvCaptureCAM_V4L::autosetup_capture_mode_v4l2() V4L2_PIX_FMT_JPEG, #endif V4L2_PIX_FMT_Y16, + V4L2_PIX_FMT_Y16_BE, V4L2_PIX_FMT_Y12, V4L2_PIX_FMT_Y10, V4L2_PIX_FMT_GREY, @@ -668,6 +677,7 @@ bool CvCaptureCAM_V4L::convertableToRgb() const case V4L2_PIX_FMT_SGBRG8: case V4L2_PIX_FMT_RGB24: case V4L2_PIX_FMT_Y16: + case V4L2_PIX_FMT_Y16_BE: case V4L2_PIX_FMT_Y10: case V4L2_PIX_FMT_GREY: case V4L2_PIX_FMT_BGR24: @@ -716,6 +726,7 @@ void CvCaptureCAM_V4L::v4l2_create_frame() size.height = size.height * 3 / 2; // "1.5" channels break; case V4L2_PIX_FMT_Y16: + case V4L2_PIX_FMT_Y16_BE: case V4L2_PIX_FMT_Y12: case V4L2_PIX_FMT_Y10: depth = IPL_DEPTH_16U; @@ -1731,8 +1742,21 @@ void CvCaptureCAM_V4L::convertToRgb(const Buffer ¤tBuffer) return; case V4L2_PIX_FMT_Y16: { + // https://www.kernel.org/doc/html/v4.10/media/uapi/v4l/pixfmt-y16.html + // This is a grey-scale image with a depth of 16 bits per pixel. The least significant byte is stored at lower memory addresses (little-endian). + // Note: 10-bits precision is not supported cv::Mat temp(imageSize, CV_8UC1, buffers[MAX_V4L_BUFFERS].memories[MEMORY_RGB].start); - cv::Mat(imageSize, CV_16UC1, start).convertTo(temp, CV_8U, 1.0 / 256); + cv::extractChannel(cv::Mat(imageSize, CV_8UC2, start), temp, 1); // 1 - second channel + cv::cvtColor(temp, destination, COLOR_GRAY2BGR); + return; + } + case V4L2_PIX_FMT_Y16_BE: + { + // https://www.kernel.org/doc/html/v4.10/media/uapi/v4l/pixfmt-y16-be.html + // This is a grey-scale image with a depth of 16 bits per pixel. The most significant byte is stored at lower memory addresses (big-endian). + // Note: 10-bits precision is not supported + cv::Mat temp(imageSize, CV_8UC1, buffers[MAX_V4L_BUFFERS].memories[MEMORY_RGB].start); + cv::extractChannel(cv::Mat(imageSize, CV_8UC2, start), temp, 0); // 0 - first channel cv::cvtColor(temp, destination, COLOR_GRAY2BGR); return; } diff --git a/modules/videoio/test/test_precomp.hpp b/modules/videoio/test/test_precomp.hpp index c39dc7da13..b4f340897e 100644 --- a/modules/videoio/test/test_precomp.hpp +++ b/modules/videoio/test/test_precomp.hpp @@ -60,7 +60,8 @@ inline std::string fourccToString(int fourcc) inline std::string fourccToStringSafe(int fourcc) { std::string res = fourccToString(fourcc); - std::replace_if(res.begin(), res.end(), [](uint8_t c){ return c < '0' || c > 'z'; }, '_'); + // TODO: return hex values for invalid characters + std::transform(res.begin(), res.end(), res.begin(), [](uint8_t c) { return (c >= '0' && c <= 'z') ? c : (c == ' ' ? '_' : 'x'); }); return res; } diff --git a/modules/videoio/test/test_v4l2.cpp b/modules/videoio/test/test_v4l2.cpp index 6763e2e33c..5d56ac097c 100644 --- a/modules/videoio/test/test_v4l2.cpp +++ b/modules/videoio/test/test_v4l2.cpp @@ -34,6 +34,13 @@ #ifndef V4L2_PIX_FMT_XBGR32 #define V4L2_PIX_FMT_XBGR32 v4l2_fourcc('X', 'R', '2', '4') #endif +#ifndef V4L2_PIX_FMT_Y16 +#define V4L2_PIX_FMT_Y16 v4l2_fourcc('Y', '1', '6', ' ') +#endif +#ifndef V4L2_PIX_FMT_Y16_BE +#define V4L2_PIX_FMT_Y16_BE v4l2_fourcc_be('Y', '1', '6', ' ') +#endif + using namespace cv; @@ -111,6 +118,7 @@ vector all_params = { // { V4L2_PIX_FMT_SGBRG8, 1, CV_8U, 1.f, 1.f }, { V4L2_PIX_FMT_RGB24, 3, CV_8U, 1.f, 1.f }, { V4L2_PIX_FMT_Y16, 1, CV_16U, 1.f, 1.f }, + { V4L2_PIX_FMT_Y16_BE, 1, CV_16U, 1.f, 1.f }, { V4L2_PIX_FMT_Y10, 1, CV_16U, 1.f, 1.f }, { V4L2_PIX_FMT_GREY, 1, CV_8U, 1.f, 1.f }, { V4L2_PIX_FMT_BGR24, 3, CV_8U, 1.f, 1.f }, From 4ee0f212cc19f7e77483d34d4cf8378945e3da31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=8D=E9=B1=BC=E5=84=BF?= <36976072+buyuer@users.noreply.github.com> Date: Fri, 14 Jul 2023 08:45:14 +0000 Subject: [PATCH 078/105] Eliminating compilation warnings when using lto in gcc12 and later versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit use -flto=auto when use gcc12 or later Signed-off-by: 不鱼儿 <36976072+buyuer@users.noreply.github.com> --- cmake/OpenCVCompilerOptions.cmake | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmake/OpenCVCompilerOptions.cmake b/cmake/OpenCVCompilerOptions.cmake index d4600943fb..8bd8668130 100644 --- a/cmake/OpenCVCompilerOptions.cmake +++ b/cmake/OpenCVCompilerOptions.cmake @@ -261,7 +261,11 @@ if(CV_GCC OR CV_CLANG) endif() if(ENABLE_LTO) - add_extra_compiler_option(-flto) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12) + add_extra_compiler_option(-flto=auto) + else() + add_extra_compiler_option(-flto) + endif() endif() if(ENABLE_THIN_LTO) add_extra_compiler_option(-flto=thin) From 192099352577d18b46840cdaf3cbf365e4c6e663 Mon Sep 17 00:00:00 2001 From: Zihao Mu Date: Fri, 14 Jul 2023 22:34:39 +0800 Subject: [PATCH 079/105] Merge pull request #23952 from zihaomu:fix_depth_conv_5x5 DNN: optimize the speed of general Depth-wise #23952 Try to solve the issue: https://github.com/opencv/opencv/issues/23941 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/perf/perf_convolution.cpp | 3 ++- modules/dnn/src/layers/cpu_kernels/convolution.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/dnn/perf/perf_convolution.cpp b/modules/dnn/perf/perf_convolution.cpp index bb890c6a00..99f425b712 100644 --- a/modules/dnn/perf/perf_convolution.cpp +++ b/modules/dnn/perf/perf_convolution.cpp @@ -744,7 +744,8 @@ static const ConvParam_t testConvolutionConfigs[] = { /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 4, 1, 1}}, 96, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 864.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 96, 1, 1}}, 4, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 772.}, /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 8, 1, 1}}, 32, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 544.}, - /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 32, 1, 1}}, 8, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 520.} + /* GFLOPS 0.000 x 1 = 0.000 */ {{1, 1}, {{1, 32, 1, 1}}, 8, 1, {1, 1}, {1, 1}, {0, 0}, {0, 0}, "SAME", true, 520.}, + /* GFLOPS 0.472 x 1 = 0.472 */ {{5, 5}, {{1, 32, 96, 96}}, 32, 32, {1, 1}, {1, 1}, {2, 2}, {0, 0}, "", true, 472154112.} }; struct ConvParamID { diff --git a/modules/dnn/src/layers/cpu_kernels/convolution.cpp b/modules/dnn/src/layers/cpu_kernels/convolution.cpp index c76b3494e2..68777ff65e 100644 --- a/modules/dnn/src/layers/cpu_kernels/convolution.cpp +++ b/modules/dnn/src/layers/cpu_kernels/convolution.cpp @@ -1290,7 +1290,7 @@ void runFastConv(InputArray _input, OutputArray _output, const Ptr& co else Kg_nblocks = 1; - bool separateIm2col = fast_1x1 || stripes_per_plane == 1; + bool separateIm2col = (fast_1x1 || stripes_per_plane == 1) && conv->conv_type != CONV_TYPE_DEPTHWISE_REMAIN; int Kstripes = Kg_nblocks * stripes_per_plane; int nsubtasks = N * ngroups * Kstripes; From e00c90458536d349f003226f3711d7dec299d600 Mon Sep 17 00:00:00 2001 From: cudawarped <12133430+cudawarped@users.noreply.github.com> Date: Mon, 17 Jul 2023 18:03:09 +0300 Subject: [PATCH 080/105] cuda: add SkipTestException handling --- modules/ts/include/opencv2/ts/cuda_test.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/ts/include/opencv2/ts/cuda_test.hpp b/modules/ts/include/opencv2/ts/cuda_test.hpp index 87b217fc13..73e8f8ba84 100644 --- a/modules/ts/include/opencv2/ts/cuda_test.hpp +++ b/modules/ts/include/opencv2/ts/cuda_test.hpp @@ -202,6 +202,11 @@ namespace cvtest { \ UnsafeTestBody(); \ } \ + catch (const cvtest::details::SkipTestExceptionBase& e) \ + { \ + printf("[ SKIP ] %s\n", e.what()); \ + cv::cuda::resetDevice(); \ + } \ catch (...) \ { \ cv::cuda::resetDevice(); \ From 55906457e60c65f0ebdd086b2eef5aa34404ef28 Mon Sep 17 00:00:00 2001 From: iarspider Date: Tue, 18 Jul 2023 09:18:17 +0200 Subject: [PATCH 081/105] test_houghlines: Fix C++20 compatibility C++20 made it invalid to use simple-template-ids for constructors and destructors: https://eel.is/c++draft/diff.cpp17.class#2 GCC 11 and later throw an error on this, with the unhelpful message `expected unqualified-id before ')' token`. This PR fixes the problem. --- modules/imgproc/test/test_houghlines.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/imgproc/test/test_houghlines.cpp b/modules/imgproc/test/test_houghlines.cpp index 61b67d9873..003420ae65 100644 --- a/modules/imgproc/test/test_houghlines.cpp +++ b/modules/imgproc/test/test_houghlines.cpp @@ -53,7 +53,7 @@ struct SimilarWith T value; float theta_eps; float rho_eps; - SimilarWith(T val, float e, float r_e): value(val), theta_eps(e), rho_eps(r_e) { }; + SimilarWith(T val, float e, float r_e): value(val), theta_eps(e), rho_eps(r_e) { }; bool operator()(const T& other); }; From 23f27d8dbe5f71aa6571b172953ead260fea49a5 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Mon, 17 Jul 2023 12:49:49 +0300 Subject: [PATCH 082/105] Use OpenCV logging instead of std::cerr. --- modules/core/src/ocl.cpp | 1 - modules/core/src/system.cpp | 6 +-- .../dnn/src/layers/max_unpooling_layer.cpp | 18 +++---- modules/dnn/src/net_impl.cpp | 16 +++--- modules/highgui/src/window_wayland.cpp | 2 +- modules/imgcodecs/src/grfmt_pxm.cpp | 6 +-- modules/imgcodecs/src/loadsave.cpp | 51 +++++++++---------- 7 files changed, 47 insertions(+), 53 deletions(-) diff --git a/modules/core/src/ocl.cpp b/modules/core/src/ocl.cpp index bf562c3092..f93a7be3f1 100644 --- a/modules/core/src/ocl.cpp +++ b/modules/core/src/ocl.cpp @@ -51,7 +51,6 @@ #include #include #include -#include // std::cerr #include #if !(defined _MSC_VER) || (defined _MSC_VER && _MSC_VER > 1700) #include diff --git a/modules/core/src/system.cpp b/modules/core/src/system.cpp index 7811ab72f0..44e1e04a03 100644 --- a/modules/core/src/system.cpp +++ b/modules/core/src/system.cpp @@ -2574,7 +2574,7 @@ public: ippStatus = ippGetCpuFeatures(&cpuFeatures, NULL); if(ippStatus < 0) { - std::cerr << "ERROR: IPP cannot detect CPU features, IPP was disabled " << std::endl; + CV_LOG_ERROR(NULL, "ERROR: IPP cannot detect CPU features, IPP was disabled"); useIPP = false; return; } @@ -2612,7 +2612,7 @@ public: if(env == "disabled") { - std::cerr << "WARNING: IPP was disabled by OPENCV_IPP environment variable" << std::endl; + CV_LOG_WARNING(NULL, "WARNING: IPP was disabled by OPENCV_IPP environment variable"); useIPP = false; } else if(env == "sse42") @@ -2626,7 +2626,7 @@ public: #endif #endif else - std::cerr << "ERROR: Improper value of OPENCV_IPP: " << env.c_str() << ". Correct values are: disabled, sse42, avx2, avx512 (Intel64 only)" << std::endl; + CV_LOG_ERROR(NULL, "ERROR: Improper value of OPENCV_IPP: " << env.c_str() << ". Correct values are: disabled, sse42, avx2, avx512 (Intel64 only)"); // Trim unsupported features ippFeatures &= cpuFeatures; diff --git a/modules/dnn/src/layers/max_unpooling_layer.cpp b/modules/dnn/src/layers/max_unpooling_layer.cpp index a44d25ce89..fd47c4c919 100644 --- a/modules/dnn/src/layers/max_unpooling_layer.cpp +++ b/modules/dnn/src/layers/max_unpooling_layer.cpp @@ -14,6 +14,7 @@ Implementation of Batch Normalization layer. #include "../op_cuda.hpp" #include "../op_halide.hpp" #include +#include #ifdef HAVE_CUDA #include "../cuda4dnn/primitives/max_unpooling.hpp" @@ -110,17 +111,12 @@ public: int index = idxptr[i_wh]; if (!(0 <= index && index < outPlaneTotal)) { - std::cerr - << "i_n=" << i_n << std::endl - << "i_c=" << i_c << std::endl - << "i_wh=" << i_wh << std::endl - << "index=" << index << std::endl - << "maxval=" << inptr[i_wh] << std::endl - << "outPlaneTotal=" << outPlaneTotal << std::endl - << "input.size=" << input.size << std::endl - << "indices.size=" << indices.size << std::endl - << "outBlob=" << outBlob.size << std::endl - ; + CV_LOG_ERROR(NULL, cv::format( + "i_n=%d\ni_c=%d\ni_wh=%d\nindex=%d\nmaxval=%lf\noutPlaneTotal=%d\n", + i_n, i_c, i_wh, index, inptr[i_wh], outPlaneTotal)); + CV_LOG_ERROR(NULL, "input.size=" << input.size); + CV_LOG_ERROR(NULL, "indices.size=" << indices.size); + CV_LOG_ERROR(NULL, "outBlob=" << outBlob.size); CV_Assert(0 <= index && index < outPlaneTotal); } outptr[index] = inptr[i_wh]; diff --git a/modules/dnn/src/net_impl.cpp b/modules/dnn/src/net_impl.cpp index c8341e4c6f..16ae6d7bfb 100644 --- a/modules/dnn/src/net_impl.cpp +++ b/modules/dnn/src/net_impl.cpp @@ -662,14 +662,14 @@ void Net::Impl::forwardLayer(LayerData& ld) m = u.getMat(ACCESS_READ); if (!checkRange(m)) { - std::cerr << "WARNING: NaN detected in layer output: id=" << ld.id << " name=" << layer->name << std::endl; - std::cerr << "output id=" << i << " output shape=" << shape(m) << std::endl; + CV_LOG_WARNING(NULL, "NaN detected in layer output: id=" << ld.id << " name=" << layer->name + << " output id=" << i << " output shape=" << shape(m)); fail = true; } else if (!checkRange(m, true, NULL, -1e6, 1e6)) { - std::cerr << "WARNING: Inf detected in layer output: id=" << ld.id << " name=" << layer->name << std::endl; - std::cerr << "output id=" << i << " output shape=" << shape(m) << std::endl; + CV_LOG_WARNING(NULL, "Inf detected in layer output: id=" << ld.id << " name=" << layer->name + << " output id=" << i << " output shape=" << shape(m)); fail = true; } } @@ -738,14 +738,14 @@ void Net::Impl::forwardLayer(LayerData& ld) const Mat& m = ld.outputBlobs[i]; if (!checkRange(m)) { - std::cerr << "WARNING: NaN detected in layer output: id=" << ld.id << " name=" << layer->name << std::endl; - std::cerr << "output id=" << i << " output shape=" << shape(m) << std::endl; + CV_LOG_WARNING(NULL, "NaN detected in layer output: " + << cv::format("id=%d name=%s output id=%zu output shape=", ld.id, layer->name.c_str(), i) << shape(m)); fail = true; } else if (!checkRange(m, true, NULL, -1e6, 1e6)) { - std::cerr << "WARNING: Inf detected in layer output: id=" << ld.id << " name=" << layer->name << std::endl; - std::cerr << "output id=" << i << " output shape=" << shape(m) << std::endl; + CV_LOG_WARNING(NULL, "Inf detected in layer output: " + << cv::format("id=%d name=%s output id=%zu output shape=", ld.id, layer->name.c_str(), i) << shape(m)); fail = true; } } diff --git a/modules/highgui/src/window_wayland.cpp b/modules/highgui/src/window_wayland.cpp index 69231c0072..0d7c9788a1 100644 --- a/modules/highgui/src/window_wayland.cpp +++ b/modules/highgui/src/window_wayland.cpp @@ -1055,7 +1055,7 @@ void cv_wl_keyboard::handle_kb_keymap(void *data, struct wl_keyboard *kb, uint32 } catch (std::exception &e) { if (keyboard->xkb_.keymap) xkb_keymap_unref(keyboard->xkb_.keymap); - std::cerr << "OpenCV Error: " << e.what() << std::endl; + CV_LOG_ERROR(NULL, "OpenCV Error: " << e.what()); } close(fd); diff --git a/modules/imgcodecs/src/grfmt_pxm.cpp b/modules/imgcodecs/src/grfmt_pxm.cpp index 8da2348728..76290c43de 100644 --- a/modules/imgcodecs/src/grfmt_pxm.cpp +++ b/modules/imgcodecs/src/grfmt_pxm.cpp @@ -43,7 +43,7 @@ #include "precomp.hpp" #include "utils.hpp" #include "grfmt_pxm.hpp" -#include +#include #ifdef HAVE_IMGCODEC_PXM @@ -191,7 +191,7 @@ bool PxMDecoder::readHeader() } catch (...) { - std::cerr << "PXM::readHeader(): unknown C++ exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "PXM::readHeader(): unknown C++ exception"); throw; } @@ -364,7 +364,7 @@ bool PxMDecoder::readData( Mat& img ) } catch (...) { - std::cerr << "PXM::readData(): unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "PXM::readData(): unknown exception"); throw; } diff --git a/modules/imgcodecs/src/loadsave.cpp b/modules/imgcodecs/src/loadsave.cpp index d0413c1ade..4ce490610e 100644 --- a/modules/imgcodecs/src/loadsave.cpp +++ b/modules/imgcodecs/src/loadsave.cpp @@ -437,12 +437,12 @@ imread_( const String& filename, int flags, Mat& mat ) } catch (const cv::Exception& e) { - std::cerr << "imread_('" << filename << "'): can't read header: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imread_('" << filename << "'): can't read header: " << e.what()); return 0; } catch (...) { - std::cerr << "imread_('" << filename << "'): can't read header: unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imread_('" << filename << "'): can't read header: unknown exception"); return 0; } @@ -475,11 +475,11 @@ imread_( const String& filename, int flags, Mat& mat ) } catch (const cv::Exception& e) { - std::cerr << "imread_('" << filename << "'): can't read data: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imread_('" << filename << "'): can't read data: " << e.what()); } catch (...) { - std::cerr << "imread_('" << filename << "'): can't read data: unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imread_('" << filename << "'): can't read data: unknown exception"); } if (!success) { @@ -542,12 +542,12 @@ imreadmulti_(const String& filename, int flags, std::vector& mats, int star } catch (const cv::Exception& e) { - std::cerr << "imreadmulti_('" << filename << "'): can't read header: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imreadmulti_('" << filename << "'): can't read header: " << e.what()); return 0; } catch (...) { - std::cerr << "imreadmulti_('" << filename << "'): can't read header: unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imreadmulti_('" << filename << "'): can't read header: unknown exception"); return 0; } @@ -591,11 +591,11 @@ imreadmulti_(const String& filename, int flags, std::vector& mats, int star } catch (const cv::Exception& e) { - std::cerr << "imreadmulti_('" << filename << "'): can't read data: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imreadmulti_('" << filename << "'): can't read data: " << e.what()); } catch (...) { - std::cerr << "imreadmulti_('" << filename << "'): can't read data: unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imreadmulti_('" << filename << "'): can't read data: unknown exception"); } if (!success) break; @@ -672,7 +672,7 @@ size_t imcount_(const String& filename, int flags) return collection.size(); } catch(cv::Exception const& e) { // Reading header or finding decoder for the filename is failed - std::cerr << "imcount_('" << filename << "'): can't read header or can't find decoder: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imcount_('" << filename << "'): can't read header or can't find decoder: " << e.what()); } return 0; } @@ -768,14 +768,13 @@ static bool imwrite_( const String& filename, const std::vector& img_vec, } catch (const cv::Exception& e) { - std::cerr << "imwrite_('" << filename << "'): can't write data: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imwrite_('" << filename << "'): can't write data: " << e.what()); } catch (...) { - std::cerr << "imwrite_('" << filename << "'): can't write data: unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imwrite_('" << filename << "'): can't write data: unknown exception"); } - // CV_Assert( code ); return code; } @@ -851,11 +850,11 @@ imdecode_( const Mat& buf, int flags, Mat& mat ) } catch (const cv::Exception& e) { - std::cerr << "imdecode_('" << filename << "'): can't read header: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imdecode_('" << filename << "'): can't read header: " << e.what()); } catch (...) { - std::cerr << "imdecode_('" << filename << "'): can't read header: unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imdecode_('" << filename << "'): can't read header: unknown exception"); } if (!success) { @@ -864,7 +863,7 @@ imdecode_( const Mat& buf, int flags, Mat& mat ) { if (0 != remove(filename.c_str())) { - std::cerr << "unable to remove temporary file:" << filename << std::endl << std::flush; + CV_LOG_WARNING(NULL, "unable to remove temporary file:" << filename); } } return 0; @@ -896,18 +895,18 @@ imdecode_( const Mat& buf, int flags, Mat& mat ) } catch (const cv::Exception& e) { - std::cerr << "imdecode_('" << filename << "'): can't read data: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imdecode_('" << filename << "'): can't read data: " << e.what()); } catch (...) { - std::cerr << "imdecode_('" << filename << "'): can't read data: unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imdecode_('" << filename << "'): can't read data: unknown exception"); } if (!filename.empty()) { if (0 != remove(filename.c_str())) { - std::cerr << "unable to remove temporary file:" << filename << std::endl << std::flush; + CV_LOG_WARNING(NULL, "unable to remove temporary file: " << filename); } } @@ -1000,11 +999,11 @@ imdecodemulti_(const Mat& buf, int flags, std::vector& mats, int start, int } catch (const cv::Exception& e) { - std::cerr << "imreadmulti_('" << filename << "'): can't read header: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imreadmulti_('" << filename << "'): can't read header: " << e.what()); } catch (...) { - std::cerr << "imreadmulti_('" << filename << "'): can't read header: unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imreadmulti_('" << filename << "'): can't read header: unknown exception"); } int current = start; @@ -1025,7 +1024,7 @@ imdecodemulti_(const Mat& buf, int flags, std::vector& mats, int start, int { if (0 != remove(filename.c_str())) { - std::cerr << "unable to remove temporary file:" << filename << std::endl << std::flush; + CV_LOG_WARNING(NULL, "unable to remove temporary file: " << filename); } } return 0; @@ -1060,11 +1059,11 @@ imdecodemulti_(const Mat& buf, int flags, std::vector& mats, int start, int } catch (const cv::Exception& e) { - std::cerr << "imreadmulti_('" << filename << "'): can't read data: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imreadmulti_('" << filename << "'): can't read data: " << e.what()); } catch (...) { - std::cerr << "imreadmulti_('" << filename << "'): can't read data: unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "imreadmulti_('" << filename << "'): can't read data: unknown exception"); } if (!success) break; @@ -1087,7 +1086,7 @@ imdecodemulti_(const Mat& buf, int flags, std::vector& mats, int start, int { if (0 != remove(filename.c_str())) { - std::cerr << "unable to remove temporary file:" << filename << std::endl << std::flush; + CV_LOG_WARNING(NULL, "unable to remove temporary file: " << filename); } } @@ -1317,10 +1316,10 @@ Mat ImageCollection::Impl::readData() { success = true; } catch (const cv::Exception &e) { - std::cerr << "ImageCollection class: can't read data: " << e.what() << std::endl << std::flush; + CV_LOG_ERROR(NULL, "ImageCollection class: can't read data: " << e.what()); } catch (...) { - std::cerr << "ImageCollection class:: can't read data: unknown exception" << std::endl << std::flush; + CV_LOG_ERROR(NULL, "ImageCollection class:: can't read data: unknown exception"); } if (!success) return cv::Mat(); From 9519e67ad21e33fb7f7ee4b5f2e04fabb89def82 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Wed, 19 Jul 2023 16:51:41 +0300 Subject: [PATCH 083/105] Merge pull request #24022 from VadimLevin:dev/vlevin/python-typing-cuda Fix python typing stubs generation for CUDA modules #24022 resolves #23946 resolves #23945 resolves opencv/opencv-python#871 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../typing_stubs_generation/api_refinement.py | 68 +++++++++++++++++-- .../src2/typing_stubs_generation/ast_utils.py | 29 +++++++- .../typing_stubs_generation/generation.py | 53 +++++---------- .../predefined_types.py | 4 ++ 4 files changed, 112 insertions(+), 42 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/api_refinement.py b/modules/python/src2/typing_stubs_generation/api_refinement.py index f7d3a9dd08..a879d0ac4b 100644 --- a/modules/python/src2/typing_stubs_generation/api_refinement.py +++ b/modules/python/src2/typing_stubs_generation/api_refinement.py @@ -2,14 +2,17 @@ __all__ = [ "apply_manual_api_refinement" ] -from typing import Sequence, Callable +from typing import cast, Sequence, Callable, Iterable -from .nodes import (NamespaceNode, FunctionNode, OptionalTypeNode, - ClassProperty, PrimitiveTypeNode) -from .ast_utils import find_function_node, SymbolName +from .nodes import (NamespaceNode, FunctionNode, OptionalTypeNode, TypeNode, + ClassProperty, PrimitiveTypeNode, ASTNodeTypeNode, + AggregatedTypeNode) +from .ast_utils import (find_function_node, SymbolName, + for_each_function_overload) def apply_manual_api_refinement(root: NamespaceNode) -> None: + refine_cuda_module(root) export_matrix_type_constants(root) # Export OpenCV exception class builtin_exception = root.add_class("Exception") @@ -57,13 +60,65 @@ def make_optional_arg(arg_name: str) -> Callable[[NamespaceNode, SymbolName], No continue overload.arguments[arg_idx].type_node = OptionalTypeNode( - overload.arguments[arg_idx].type_node + cast(TypeNode, overload.arguments[arg_idx].type_node) ) return _make_optional_arg -def _find_argument_index(arguments: Sequence[FunctionNode.Arg], name: str) -> int: +def refine_cuda_module(root: NamespaceNode) -> None: + def fix_cudaoptflow_enums_names() -> None: + for class_name in ("NvidiaOpticalFlow_1_0", "NvidiaOpticalFlow_2_0"): + if class_name not in cuda_root.classes: + continue + opt_flow_class = cuda_root.classes[class_name] + _trim_class_name_from_argument_types( + for_each_function_overload(opt_flow_class), class_name + ) + + def fix_namespace_usage_scope(cuda_ns: NamespaceNode) -> None: + USED_TYPES = ("GpuMat", "Stream") + + def fix_type_usage(type_node: TypeNode) -> None: + if isinstance(type_node, AggregatedTypeNode): + for item in type_node.items: + fix_type_usage(item) + if isinstance(type_node, ASTNodeTypeNode): + if type_node._typename in USED_TYPES: + type_node._typename = f"cuda_{type_node._typename}" + + for overload in for_each_function_overload(cuda_ns): + if overload.return_type is not None: + fix_type_usage(overload.return_type.type_node) + for type_node in [arg.type_node for arg in overload.arguments + if arg.type_node is not None]: + fix_type_usage(type_node) + + if "cuda" not in root.namespaces: + return + cuda_root = root.namespaces["cuda"] + fix_cudaoptflow_enums_names() + for ns in [ns for ns_name, ns in root.namespaces.items() + if ns_name.startswith("cuda")]: + fix_namespace_usage_scope(ns) + + +def _trim_class_name_from_argument_types( + overloads: Iterable[FunctionNode.Overload], + class_name: str +) -> None: + separator = f"{class_name}_" + for overload in overloads: + for arg in [arg for arg in overload.arguments + if arg.type_node is not None]: + ast_node = cast(ASTNodeTypeNode, arg.type_node) + if class_name in ast_node.ctype_name: + fixed_name = ast_node._typename.split(separator)[-1] + ast_node._typename = fixed_name + + +def _find_argument_index(arguments: Sequence[FunctionNode.Arg], + name: str) -> int: for i, arg in enumerate(arguments): if arg.name == name: return i @@ -76,6 +131,7 @@ NODES_TO_REFINE = { SymbolName(("cv", ), (), "resize"): make_optional_arg("dsize"), SymbolName(("cv", ), (), "calcHist"): make_optional_arg("mask"), } + ERROR_CLASS_PROPERTIES = ( ClassProperty("code", PrimitiveTypeNode.int_(), False), ClassProperty("err", PrimitiveTypeNode.str_(), False), diff --git a/modules/python/src2/typing_stubs_generation/ast_utils.py b/modules/python/src2/typing_stubs_generation/ast_utils.py index 4cdf807260..47c06571a5 100644 --- a/modules/python/src2/typing_stubs_generation/ast_utils.py +++ b/modules/python/src2/typing_stubs_generation/ast_utils.py @@ -1,5 +1,5 @@ from typing import (NamedTuple, Sequence, Tuple, Union, List, - Dict, Callable, Optional) + Dict, Callable, Optional, Generator) import keyword from .nodes import (ASTNode, NamespaceNode, ClassNode, FunctionNode, @@ -404,6 +404,33 @@ def get_enum_module_and_export_name(enum_node: EnumerationNode) -> Tuple[str, st return enum_export_name, namespace_node.full_export_name +def for_each_class( + node: Union[NamespaceNode, ClassNode] +) -> Generator[ClassNode, None, None]: + for cls in node.classes.values(): + yield cls + if len(cls.classes): + yield from for_each_class(cls) + + +def for_each_function( + node: Union[NamespaceNode, ClassNode], + traverse_class_nodes: bool = True +) -> Generator[FunctionNode, None, None]: + yield from node.functions.values() + if traverse_class_nodes: + for cls in for_each_class(node): + yield from for_each_function(cls) + + +def for_each_function_overload( + node: Union[NamespaceNode, ClassNode], + traverse_class_nodes: bool = True +) -> Generator[FunctionNode.Overload, None, None]: + for func in for_each_function(node, traverse_class_nodes): + yield from func.overloads + + if __name__ == '__main__': import doctest doctest.testmod() diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index 2d6d4d338e..b4e8cb22c6 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -3,17 +3,20 @@ __all__ = ("generate_typing_stubs", ) from io import StringIO from pathlib import Path import re -from typing import (Generator, Type, Callable, NamedTuple, Union, Set, Dict, +from typing import (Type, Callable, NamedTuple, Union, Set, Dict, Collection, Tuple, List) import warnings -from .ast_utils import get_enclosing_namespace, get_enum_module_and_export_name +from .ast_utils import (get_enclosing_namespace, + get_enum_module_and_export_name, + for_each_function_overload, + for_each_class) from .predefined_types import PREDEFINED_TYPES from .api_refinement import apply_manual_api_refinement -from .nodes import (ASTNode, ASTNodeType, NamespaceNode, ClassNode, FunctionNode, - EnumerationNode, ConstantNode) +from .nodes import (ASTNode, ASTNodeType, NamespaceNode, ClassNode, + FunctionNode, EnumerationNode, ConstantNode) from .nodes.type_node import (TypeNode, AliasTypeNode, AliasRefTypeNode, AggregatedTypeNode, ASTNodeTypeNode, @@ -105,8 +108,9 @@ def _generate_typing_stubs(root: NamespaceNode, output_path: Path) -> None: # NOTE: Enumerations require special handling, because all enumeration # constants are exposed as module attributes - has_enums = _generate_section_stub(StubSection("# Enumerations", EnumerationNode), - root, output_stream, 0) + has_enums = _generate_section_stub( + StubSection("# Enumerations", EnumerationNode), root, output_stream, 0 + ) # Collect all enums from class level and export them to module level for class_node in root.classes.values(): if _generate_enums_from_classes_tree(class_node, output_stream, indent=0): @@ -536,30 +540,6 @@ def check_overload_presence(node: Union[NamespaceNode, ClassNode]) -> bool: return True return False - -def _for_each_class(node: Union[NamespaceNode, ClassNode]) \ - -> Generator[ClassNode, None, None]: - for cls in node.classes.values(): - yield cls - if len(cls.classes): - yield from _for_each_class(cls) - - -def _for_each_function(node: Union[NamespaceNode, ClassNode]) \ - -> Generator[FunctionNode, None, None]: - for func in node.functions.values(): - yield func - for cls in node.classes.values(): - yield from _for_each_function(cls) - - -def _for_each_function_overload(node: Union[NamespaceNode, ClassNode]) \ - -> Generator[FunctionNode.Overload, None, None]: - for func in _for_each_function(node): - for overload in func.overloads: - yield overload - - def _collect_required_imports(root: NamespaceNode) -> Set[str]: """Collects all imports required for classes and functions typing stubs declarations. @@ -582,7 +562,7 @@ def _collect_required_imports(root: NamespaceNode) -> Set[str]: has_overload = check_overload_presence(root) # if there is no module-level functions with overload, check its presence # during class traversing, including their inner-classes - for cls in _for_each_class(root): + for cls in for_each_class(root): if not has_overload and check_overload_presence(cls): has_overload = True required_imports.add("import typing") @@ -600,8 +580,9 @@ def _collect_required_imports(root: NamespaceNode) -> Set[str]: if has_overload: required_imports.add("import typing") # Importing modules required to resolve functions arguments - for overload in _for_each_function_overload(root): - for arg in filter(lambda a: a.type_node is not None, overload.arguments): + for overload in for_each_function_overload(root): + for arg in filter(lambda a: a.type_node is not None, + overload.arguments): _add_required_usage_imports(arg.type_node, required_imports) # type: ignore if overload.return_type is not None: _add_required_usage_imports(overload.return_type.type_node, @@ -625,11 +606,13 @@ def _populate_reexported_symbols(root: NamespaceNode) -> None: _reexport_submodule(root) - # Special cases, symbols defined in possible pure Python submodules should be + # Special cases, symbols defined in possible pure Python submodules + # should be root.reexported_submodules_symbols["mat_wrapper"].append("Mat") -def _write_reexported_symbols_section(module: NamespaceNode, output_stream: StringIO) -> None: +def _write_reexported_symbols_section(module: NamespaceNode, + output_stream: StringIO) -> None: """Write re-export section for the given module. Re-export statements have from `from module_name import smth as smth`. diff --git a/modules/python/src2/typing_stubs_generation/predefined_types.py b/modules/python/src2/typing_stubs_generation/predefined_types.py index fe9a37a45e..ce4f901e79 100644 --- a/modules/python/src2/typing_stubs_generation/predefined_types.py +++ b/modules/python/src2/typing_stubs_generation/predefined_types.py @@ -22,6 +22,10 @@ _PREDEFINED_TYPES = ( PrimitiveTypeNode.int_("uchar"), PrimitiveTypeNode.int_("unsigned"), PrimitiveTypeNode.int_("int64"), + PrimitiveTypeNode.int_("uint8_t"), + PrimitiveTypeNode.int_("int8_t"), + PrimitiveTypeNode.int_("int32_t"), + PrimitiveTypeNode.int_("uint32_t"), PrimitiveTypeNode.int_("size_t"), PrimitiveTypeNode.float_("float"), PrimitiveTypeNode.float_("double"), From 4c568e6ed3c58ffbbfcc47f841cfdea1eaa8b8b8 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Wed, 19 Jul 2023 17:22:10 +0300 Subject: [PATCH 084/105] fix: preserve NumPY writeable flag in output arguments --- modules/python/src2/cv2_convert.cpp | 7 +++++++ modules/python/test/test_misc.py | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/modules/python/src2/cv2_convert.cpp b/modules/python/src2/cv2_convert.cpp index 20cd957753..e9e1fed4fd 100644 --- a/modules/python/src2/cv2_convert.cpp +++ b/modules/python/src2/cv2_convert.cpp @@ -101,6 +101,13 @@ bool pyopencv_to(PyObject* o, Mat& m, const ArgInfo& info) PyArrayObject* oarr = (PyArrayObject*) o; + if (info.outputarg && !PyArray_ISWRITEABLE(oarr)) + { + failmsg("%s marked as output argument, but provided NumPy array " + "marked as readonly", info.name); + return false; + } + bool needcopy = false, needcast = false; int typenum = PyArray_TYPE(oarr), new_typenum = typenum; int type = typenum == NPY_UBYTE ? CV_8U : diff --git a/modules/python/test/test_misc.py b/modules/python/test/test_misc.py index afe3fcc5db..9f7406587c 100644 --- a/modules/python/test/test_misc.py +++ b/modules/python/test/test_misc.py @@ -232,6 +232,12 @@ class Arguments(NewOpenCVTests): Exception, msg, cv.utils.dumpInputArray, test_array ) + def test_numpy_writeable_flag_is_preserved(self): + array = np.zeros((10, 10, 1), dtype=np.uint8) + array.setflags(write=False) + with self.assertRaises(Exception): + cv.rectangle(array, (0, 0), (5, 5), (255), 2) + def test_20968(self): pixel = np.uint8([[[40, 50, 200]]]) _ = cv.cvtColor(pixel, cv.COLOR_RGB2BGR) # should not raise exception From 1794cdc03c9505bb46f33a5cde5e210c1c7f65a4 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Thu, 20 Jul 2023 09:18:29 +0300 Subject: [PATCH 085/105] Merge pull request #24023 from VadimLevin:dev/vlevin/python-typing-magic-constants Python typing magic constants #24023 This patch adds typing stubs generation for `__all__` and `__version__` constants. Introduced `__all__` is intentionally empty for all generated modules stubs. Type hints won't work for star imports resolves #23950 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../python/src2/typing_stubs_generation/api_refinement.py | 2 ++ modules/python/src2/typing_stubs_generation/generation.py | 7 +++++++ .../src2/typing_stubs_generation/nodes/constant_node.py | 3 ++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/modules/python/src2/typing_stubs_generation/api_refinement.py b/modules/python/src2/typing_stubs_generation/api_refinement.py index a879d0ac4b..c3446e7a45 100644 --- a/modules/python/src2/typing_stubs_generation/api_refinement.py +++ b/modules/python/src2/typing_stubs_generation/api_refinement.py @@ -20,6 +20,8 @@ def apply_manual_api_refinement(root: NamespaceNode) -> None: root.add_class("error", (builtin_exception, ), ERROR_CLASS_PROPERTIES) for symbol_name, refine_symbol in NODES_TO_REFINE.items(): refine_symbol(root, symbol_name) + version_constant = root.add_constant("__version__", "") + version_constant._value_type = "str" def export_matrix_type_constants(root: NamespaceNode) -> None: diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index b4e8cb22c6..41525d4114 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -101,6 +101,9 @@ def _generate_typing_stubs(root: NamespaceNode, output_path: Path) -> None: output_stream = StringIO() + # Add empty __all__ dunder on top of the module + output_stream.write("__all__: list[str] = []\n\n") + # Write required imports at the top of file _write_required_imports(required_imports, output_stream) @@ -318,6 +321,10 @@ def _generate_constant_stub(constant_node: ConstantNode, export_name = extra_export_prefix + constant_node.export_name write_constant_to_stream(export_name) if generate_uppercase_version: + # Handle Python "magic" constants like __version__ + if re.match(r"^__.*__$", export_name) is not None: + return export_name, + uppercase_name = re.sub(r"([a-z])([A-Z])", r"\1_\2", export_name).upper() if export_name != uppercase_name: write_constant_to_stream(uppercase_name) diff --git a/modules/python/src2/typing_stubs_generation/nodes/constant_node.py b/modules/python/src2/typing_stubs_generation/nodes/constant_node.py index 63abd8bfb4..3dae255a3c 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/constant_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/constant_node.py @@ -11,6 +11,7 @@ class ConstantNode(ASTNode): export_name: Optional[str] = None) -> None: super().__init__(name, parent, export_name) self.value = value + self._value_type = "int" @property def children_types(self) -> Tuple[Type[ASTNode], ...]: @@ -22,7 +23,7 @@ class ConstantNode(ASTNode): @property def value_type(self) -> str: - return 'int' + return self._value_type def __str__(self) -> str: return "Constant('{}' exported as '{}': {})".format( From 20784d3da2eb1bb22123b8bbf2ffa8a721c5a5f7 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Thu, 20 Jul 2023 15:29:34 +0200 Subject: [PATCH 086/105] Fix undefined behavior with wrong function pointers called. Details here: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=58006 runtime error: call to function (unknown) through pointer to incorrect function type 'void (*)(const unsigned char **, const int *, unsigned char **, const int *, int, int)' --- modules/core/src/channels.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/core/src/channels.cpp b/modules/core/src/channels.cpp index 6ceed44a28..fd6600686d 100644 --- a/modules/core/src/channels.cpp +++ b/modules/core/src/channels.cpp @@ -53,25 +53,25 @@ static void mixChannels8u( const uchar** src, const int* sdelta, mixChannels_(src, sdelta, dst, ddelta, len, npairs); } -static void mixChannels16u( const ushort** src, const int* sdelta, - ushort** dst, const int* ddelta, +static void mixChannels16u( const uchar** src, const int* sdelta, + uchar** dst, const int* ddelta, int len, int npairs ) { - mixChannels_(src, sdelta, dst, ddelta, len, npairs); + mixChannels_((const ushort**)src, sdelta, (ushort**)dst, ddelta, len, npairs); } -static void mixChannels32s( const int** src, const int* sdelta, - int** dst, const int* ddelta, +static void mixChannels32s( const uchar** src, const int* sdelta, + uchar** dst, const int* ddelta, int len, int npairs ) { - mixChannels_(src, sdelta, dst, ddelta, len, npairs); + mixChannels_((const int**)src, sdelta, (int**)dst, ddelta, len, npairs); } -static void mixChannels64s( const int64** src, const int* sdelta, - int64** dst, const int* ddelta, +static void mixChannels64s( const uchar** src, const int* sdelta, + uchar** dst, const int* ddelta, int len, int npairs ) { - mixChannels_(src, sdelta, dst, ddelta, len, npairs); + mixChannels_((const int64**)src, sdelta, (int64**)dst, ddelta, len, npairs); } typedef void (*MixChannelsFunc)( const uchar** src, const int* sdelta, @@ -81,9 +81,9 @@ static MixChannelsFunc getMixchFunc(int depth) { static MixChannelsFunc mixchTab[] = { - (MixChannelsFunc)mixChannels8u, (MixChannelsFunc)mixChannels8u, (MixChannelsFunc)mixChannels16u, - (MixChannelsFunc)mixChannels16u, (MixChannelsFunc)mixChannels32s, (MixChannelsFunc)mixChannels32s, - (MixChannelsFunc)mixChannels64s, 0 + mixChannels8u, mixChannels8u, mixChannels16u, + mixChannels16u, mixChannels32s, mixChannels32s, + mixChannels64s, 0 }; return mixchTab[depth]; From 423ab8ddb8bf039e69ebbf98a53925562ad6962a Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Thu, 20 Jul 2023 15:53:57 +0200 Subject: [PATCH 087/105] Use void* --- modules/core/src/channels.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/core/src/channels.cpp b/modules/core/src/channels.cpp index fd6600686d..efaeb91068 100644 --- a/modules/core/src/channels.cpp +++ b/modules/core/src/channels.cpp @@ -46,36 +46,36 @@ mixChannels_( const T** src, const int* sdelta, } -static void mixChannels8u( const uchar** src, const int* sdelta, - uchar** dst, const int* ddelta, +static void mixChannels8u( const void** src, const int* sdelta, + void** dst, const int* ddelta, int len, int npairs ) { - mixChannels_(src, sdelta, dst, ddelta, len, npairs); + mixChannels_((const uchar**)src, sdelta, (uchar**)dst, ddelta, len, npairs); } -static void mixChannels16u( const uchar** src, const int* sdelta, - uchar** dst, const int* ddelta, +static void mixChannels16u( const void** src, const int* sdelta, + void** dst, const int* ddelta, int len, int npairs ) { mixChannels_((const ushort**)src, sdelta, (ushort**)dst, ddelta, len, npairs); } -static void mixChannels32s( const uchar** src, const int* sdelta, - uchar** dst, const int* ddelta, +static void mixChannels32s( const void** src, const int* sdelta, + void** dst, const int* ddelta, int len, int npairs ) { mixChannels_((const int**)src, sdelta, (int**)dst, ddelta, len, npairs); } -static void mixChannels64s( const uchar** src, const int* sdelta, - uchar** dst, const int* ddelta, +static void mixChannels64s( const void** src, const int* sdelta, + void** dst, const int* ddelta, int len, int npairs ) { mixChannels_((const int64**)src, sdelta, (int64**)dst, ddelta, len, npairs); } -typedef void (*MixChannelsFunc)( const uchar** src, const int* sdelta, - uchar** dst, const int* ddelta, int len, int npairs ); +typedef void (*MixChannelsFunc)( const void** src, const int* sdelta, + void** dst, const int* ddelta, int len, int npairs ); static MixChannelsFunc getMixchFunc(int depth) { @@ -158,7 +158,7 @@ void cv::mixChannels( const Mat* src, size_t nsrcs, Mat* dst, size_t ndsts, cons for( int t = 0; t < total; t += blocksize ) { int bsz = std::min(total - t, blocksize); - func( srcs, sdelta, dsts, ddelta, bsz, (int)npairs ); + func( (const void**)srcs, sdelta, (void **)dsts, ddelta, bsz, (int)npairs ); if( t + blocksize < total ) for( k = 0; k < npairs; k++ ) From e41ba90f17d646bb6fc3e8acbc3332bc0423c817 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Fri, 21 Jul 2023 09:13:37 +0300 Subject: [PATCH 088/105] Merge pull request #24004 from dkurt:tflite_new_layers [TFLite] Pack layer and other fixes for SSD from Keras #24004 ### Pull Request Readiness Checklist resolves https://github.com/opencv/opencv/issues/23992 **Merge with extra**: https://github.com/opencv/opencv_extra/pull/1076 See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/tflite/tflite_importer.cpp | 105 ++++++++++++++++++--- modules/dnn/test/test_tflite_importer.cpp | 20 +++- 2 files changed, 111 insertions(+), 14 deletions(-) diff --git a/modules/dnn/src/tflite/tflite_importer.cpp b/modules/dnn/src/tflite/tflite_importer.cpp index 4a186eaee0..7feded69ce 100644 --- a/modules/dnn/src/tflite/tflite_importer.cpp +++ b/modules/dnn/src/tflite/tflite_importer.cpp @@ -59,6 +59,7 @@ private: void parseUnpooling(const Operator& op, const std::string& opcode, LayerParams& layerParams); void parseReshape(const Operator& op, const std::string& opcode, LayerParams& layerParams); void parseConcat(const Operator& op, const std::string& opcode, LayerParams& layerParams); + void parsePack(const Operator& op, const std::string& opcode, LayerParams& layerParams); void parseResize(const Operator& op, const std::string& opcode, LayerParams& layerParams); void parseDeconvolution(const Operator& op, const std::string& opcode, LayerParams& layerParams); void parseQuantize(const Operator& op, const std::string& opcode, LayerParams& layerParams); @@ -70,6 +71,8 @@ private: void parseActivation(const Operator& op, const std::string& opcode, LayerParams& layerParams, bool isFused); void addLayer(LayerParams& layerParams, const Operator& op); int addPermuteLayer(const std::vector& order, const std::string& permName, const std::pair& inpId, int dtype); + int addReshapeLayer(const std::vector& shape, int axis, int num_axes, + const std::string& name, const std::pair& inpId, int dtype); inline bool isInt8(const Operator& op); inline void getQuantParams(const Operator& op, float& inpScale, int& inpZero, float& outScale, int& outZero); }; @@ -267,6 +270,7 @@ TFLiteImporter::DispatchMap TFLiteImporter::buildDispatchMap() dispatch["PAD"] = &TFLiteImporter::parsePadding; dispatch["RESHAPE"] = &TFLiteImporter::parseReshape; dispatch["CONCATENATION"] = &TFLiteImporter::parseConcat; + dispatch["PACK"] = &TFLiteImporter::parsePack; dispatch["RESIZE_BILINEAR"] = dispatch["RESIZE_NEAREST_NEIGHBOR"] = &TFLiteImporter::parseResize; dispatch["Convolution2DTransposeBias"] = &TFLiteImporter::parseDeconvolution; dispatch["QUANTIZE"] = &TFLiteImporter::parseQuantize; @@ -596,16 +600,6 @@ void TFLiteImporter::parseUnpooling(const Operator& op, const std::string& opcod void TFLiteImporter::parseReshape(const Operator& op, const std::string& opcode, LayerParams& layerParams) { DataLayout inpLayout = layouts[op.inputs()->Get(0)]; - if (inpLayout == DNN_LAYOUT_NHWC) { - // Permute to NCHW - std::vector order = {0, 2, 3, 1}; - const std::string name = layerParams.name + "/permute"; - auto inpId = layerIds[op.inputs()->Get(0)]; - int permId = addPermuteLayer(order, name, inpId, isInt8(op) ? CV_8S : CV_32F); // NCHW -> NHWC - layerIds[op.inputs()->Get(0)] = std::make_pair(permId, 0); - layouts[op.outputs()->Get(0)] = DNN_LAYOUT_NCHW; - } - layerParams.type = "Reshape"; std::vector shape; if (op.inputs()->size() > 1) { @@ -615,6 +609,22 @@ void TFLiteImporter::parseReshape(const Operator& op, const std::string& opcode, CV_Assert(options); shape.assign(options->new_shape()->begin(), options->new_shape()->end()); } + + if (inpLayout == DNN_LAYOUT_NHWC) { + if (shape.size() == 4) { + // Keep data but change a shape to OpenCV's NCHW order + std::swap(shape[2], shape[3]); + std::swap(shape[1], shape[2]); + } else { + // Permute to NCHW entire data and reshape to given a shape + std::vector order = {0, 2, 3, 1}; + const std::string name = layerParams.name + "/permute"; + auto inpId = layerIds[op.inputs()->Get(0)]; + int permId = addPermuteLayer(order, name, inpId, isInt8(op) ? CV_8S : CV_32F); // NCHW -> NHWC + layerIds[op.inputs()->Get(0)] = std::make_pair(permId, 0); + layouts[op.outputs()->Get(0)] = DNN_LAYOUT_NCHW; + } + } layerParams.set("dim", DictValue::arrayInt(shape.data(), shape.size())); addLayer(layerParams, op); } @@ -636,6 +646,47 @@ void TFLiteImporter::parseConcat(const Operator& op, const std::string& opcode, parseFusedActivation(op, options->fused_activation_function()); } +void TFLiteImporter::parsePack(const Operator& op, const std::string& opcode, LayerParams& layerParams) { + auto options = reinterpret_cast(op.builtin_options()); + int axis = options->axis(); + + DataLayout inpLayout = layouts[op.inputs()->Get(0)]; + if (inpLayout == DNN_LAYOUT_NHWC) { + // OpenCV works in NCHW data layout. So change the axis correspondingly. + axis = normalize_axis(axis, 5); // 5 because Pack adds a new axis so -1 would mean 4 + static const int remap[] = {0, 1, 3, 4, 2}; + axis = remap[axis]; + } + + // Replace Pack layer to Reshape + Concat + // Use a set because there are models which replicate single layer data by Pack. + std::set op_inputs(op.inputs()->begin(), op.inputs()->end()); + std::map > originLayerIds; + for (int inp : op_inputs) { + auto inpId = layerIds[inp]; + int dims = modelTensors->Get(inp)->shape()->size(); + + std::vector shape{1, -1}; + if (axis == dims) { + std::swap(shape[0], shape[1]); + } + const auto name = modelTensors->Get(inp)->name()->str() + "/reshape"; + int reshapeId = addReshapeLayer(shape, axis == dims ? dims - 1 : axis, 1, + name, inpId, isInt8(op) ? CV_8S : CV_32F); + + originLayerIds[inp] = layerIds[inp]; + layerIds[inp] = std::make_pair(reshapeId, 0); + } + layerParams.type = "Concat"; + layerParams.set("axis", axis); + addLayer(layerParams, op); + + // Restore origin layer inputs + for (const auto& ids : originLayerIds) { + layerIds[ids.first] = ids.second; + } +} + void TFLiteImporter::parseResize(const Operator& op, const std::string& opcode, LayerParams& layerParams) { layerParams.type = "Resize"; @@ -666,6 +717,18 @@ int TFLiteImporter::addPermuteLayer(const std::vector& order, const std::st return permId; } +int TFLiteImporter::addReshapeLayer(const std::vector& shape, int axis, int num_axes, + const std::string& name, const std::pair& inpId, int dtype) +{ + LayerParams lp; + lp.set("axis", axis); + lp.set("dim", DictValue::arrayInt(shape.data(), shape.size())); + lp.set("num_axes", num_axes); + int id = dstNet.addLayer(name, "Reshape", dtype, lp); + dstNet.connect(inpId.first, inpId.second, id, 0); + return id; +} + void TFLiteImporter::parseDeconvolution(const Operator& op, const std::string& opcode, LayerParams& layerParams) { layerParams.type = "Deconvolution"; @@ -771,6 +834,8 @@ void TFLiteImporter::parseDetectionPostProcess(const Operator& op, const std::st parameters[keys[i]] = *reinterpret_cast(data + offset + i * 4); } + parameters["num_classes"] = modelTensors->Get(op.inputs()->Get(1))->shape()->Get(2); + layerParams.type = "DetectionOutput"; layerParams.set("num_classes", parameters["num_classes"]); layerParams.set("share_location", true); @@ -780,7 +845,6 @@ void TFLiteImporter::parseDetectionPostProcess(const Operator& op, const std::st layerParams.set("top_k", parameters["max_detections"]); layerParams.set("keep_top_k", parameters["max_detections"]); layerParams.set("code_type", "CENTER_SIZE"); - layerParams.set("variance_encoded_in_target", true); layerParams.set("loc_pred_transposed", true); // Replace third input from tensor to Const layer with the priors @@ -796,10 +860,27 @@ void TFLiteImporter::parseDetectionPostProcess(const Operator& op, const std::st priors.col(2) = priors.col(0) + priors.col(3); priors.col(3) = priors.col(1) + tmp; + float x_scale = *(float*)¶meters["x_scale"]; + float y_scale = *(float*)¶meters["y_scale"]; + float w_scale = *(float*)¶meters["w_scale"]; + float h_scale = *(float*)¶meters["h_scale"]; + if (x_scale != 1.0f || y_scale != 1.0f || w_scale != 1.0f || h_scale != 1.0f) { + int numPriors = priors.rows; + priors.resize(numPriors * 2); + Mat_ scales({1, 4}, {1.f / x_scale, 1.f / y_scale, + 1.f / w_scale, 1.f / h_scale}); + repeat(scales, numPriors, 1, priors.rowRange(numPriors, priors.rows)); + priors = priors.reshape(1, {1, 2, (int)priors.total() / 2}); + layerParams.set("variance_encoded_in_target", false); + } else { + priors = priors.reshape(1, {1, 1, (int)priors.total()}); + layerParams.set("variance_encoded_in_target", true); + } + LayerParams priorsLP; priorsLP.name = layerParams.name + "/priors"; priorsLP.type = "Const"; - priorsLP.blobs.resize(1, priors.reshape(1, {1, 1, (int)priors.total()})); + priorsLP.blobs.resize(1, priors); int priorsId = dstNet.addLayer(priorsLP.name, priorsLP.type, priorsLP); layerIds[op.inputs()->Get(2)] = std::make_pair(priorsId, 0); diff --git a/modules/dnn/test/test_tflite_importer.cpp b/modules/dnn/test/test_tflite_importer.cpp index 5a1742ed97..c5bee0c086 100644 --- a/modules/dnn/test/test_tflite_importer.cpp +++ b/modules/dnn/test/test_tflite_importer.cpp @@ -31,9 +31,8 @@ void testInputShapes(const Net& net, const std::vector& inps) { } } -void testModel(const std::string& modelName, const Mat& input, double l1 = 1e-5, double lInf = 1e-4) +void testModel(Net& net, const std::string& modelName, const Mat& input, double l1 = 1e-5, double lInf = 1e-4) { - Net net = readNet(findDataFile("dnn/tflite/" + modelName + ".tflite", false)); testInputShapes(net, {input}); net.setInput(input); @@ -49,6 +48,12 @@ void testModel(const std::string& modelName, const Mat& input, double l1 = 1e-5, } } +void testModel(const std::string& modelName, const Mat& input, double l1 = 1e-5, double lInf = 1e-4) +{ + Net net = readNet(findDataFile("dnn/tflite/" + modelName + ".tflite", false)); + testModel(net, modelName, input, l1, lInf); +} + void testModel(const std::string& modelName, const Size& inpSize, double l1 = 1e-5, double lInf = 1e-4) { Mat input = imread(findDataFile("cv/shared/lena.png")); @@ -56,6 +61,13 @@ void testModel(const std::string& modelName, const Size& inpSize, double l1 = 1e testModel(modelName, input, l1, lInf); } +void testLayer(const std::string& modelName, double l1 = 1e-5, double lInf = 1e-4) +{ + Mat inp = blobFromNPY(findDataFile("dnn/tflite/" + modelName + "_inp.npy")); + Net net = readNet(findDataFile("dnn/tflite/" + modelName + ".tflite")); + testModel(net, modelName, inp, l1, lInf); +} + // https://google.github.io/mediapipe/solutions/face_mesh TEST(Test_TFLite, face_landmark) { @@ -146,6 +158,10 @@ TEST(Test_TFLite, EfficientDet_int8) { normAssertDetections(ref, out, "", 0.5, 0.05, 0.1); } +TEST(Test_TFLite, replicate_by_pack) { + testLayer("replicate_by_pack"); +} + }} // namespace #endif // OPENCV_TEST_DNN_TFLITE From 3cce299a789d3d504146ea73dd7f5e6dec474dd4 Mon Sep 17 00:00:00 2001 From: Clement Courbet Date: Mon, 17 Jul 2023 14:40:57 +0200 Subject: [PATCH 089/105] Use intrinsics for `cvRound` on x86 and x86_64 `__GNUC__` (clang/gcc linux) too. We've measured a 7x improvement in speed for `cvRound` using the intrinsic. --- modules/core/include/opencv2/core/fast_math.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/core/include/opencv2/core/fast_math.hpp b/modules/core/include/opencv2/core/fast_math.hpp index 47a2948222..9ec984d7aa 100644 --- a/modules/core/include/opencv2/core/fast_math.hpp +++ b/modules/core/include/opencv2/core/fast_math.hpp @@ -201,7 +201,7 @@ cvRound( double value ) { #if defined CV_INLINE_ROUND_DBL CV_INLINE_ROUND_DBL(value); -#elif (defined _MSC_VER && defined _M_X64) && !defined(__CUDACC__) +#elif ((defined _MSC_VER && defined _M_X64) || (defined __GNUC__ && defined __SSE2__)) && !defined(__CUDACC__) __m128d t = _mm_set_sd( value ); return _mm_cvtsd_si32(t); #elif defined _MSC_VER && defined _M_IX86 @@ -323,7 +323,7 @@ CV_INLINE int cvRound(float value) { #if defined CV_INLINE_ROUND_FLT CV_INLINE_ROUND_FLT(value); -#elif (defined _MSC_VER && defined _M_X64) && !defined(__CUDACC__) +#elif ((defined _MSC_VER && defined _M_X64) || (defined __GNUC__ && defined __SSE2__)) && !defined(__CUDACC__) __m128 t = _mm_set_ss( value ); return _mm_cvtss_si32(t); #elif defined _MSC_VER && defined _M_IX86 From 5261961a6eb9b5572e2e3b5cb0849531e2cfc1e0 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Fri, 21 Jul 2023 10:18:06 +0100 Subject: [PATCH 090/105] Merge pull request #24024 from TolyaTalamanov:at/add-onnx-openvino-execution-provider G-API: Support OpenVINO Execution Provider for ONNXRT Backend #24024 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- .../opencv2/gapi/infer/bindings_onnx.hpp | 6 + .../gapi/include/opencv2/gapi/infer/onnx.hpp | 149 +++++++++++++++++- modules/gapi/misc/python/pyopencv_gapi.hpp | 1 + .../gapi/src/backends/onnx/bindings_onnx.cpp | 12 ++ .../gapi/src/backends/onnx/gonnxbackend.cpp | 41 ++++- 5 files changed, 207 insertions(+), 2 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp index af9f3c6f6f..0f764fe2cd 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp @@ -33,6 +33,12 @@ public: GAPI_WRAP PyParams& cfgNormalize(const std::string &layer_name, bool flag); + GAPI_WRAP + PyParams& cfgExecutionProvider(ep::OpenVINO ov_ep); + + GAPI_WRAP + PyParams& cfgDisableMemPattern(); + GBackend backend() const; std::string tag() const; cv::util::any params() const; diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp index dc9a51e541..9c2118c3ad 100644 --- a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -27,6 +27,118 @@ namespace gapi { */ namespace onnx { +/** + * @brief This namespace contains Execution Providers structures for G-API ONNX Runtime backend. + */ +namespace ep { + +/** + * @brief This structure provides functions + * that fill inference options for ONNX OpenVINO Execution Provider. + * Please follow https://onnxruntime.ai/docs/execution-providers/OpenVINO-ExecutionProvider.html#summary-of-options + */ +struct GAPI_EXPORTS_W_SIMPLE OpenVINO { + // NB: Used from python. + /// @private -- Exclude this constructor from OpenCV documentation + GAPI_WRAP + OpenVINO() = default; + + /** @brief Class constructor. + + Constructs OpenVINO parameters based on device information. + + @param device Target device to use. + */ + GAPI_WRAP + OpenVINO(const std::string &device) + : device_id(device) { + } + + /** @brief Specifies OpenVINO Execution Provider device type. + + This function is used to override the accelerator hardware type + and precision at runtime. If this option is not explicitly configured, default + hardware and precision specified during onnxruntime build time is used. + + @param type Device type ("CPU_FP32", "GPU_FP16", etc) + @return reference to this parameter structure. + */ + GAPI_WRAP + OpenVINO& cfgDeviceType(const std::string &type) { + device_type = cv::util::make_optional(type); + return *this; + } + + /** @brief Specifies OpenVINO Execution Provider cache dir. + + This function is used to explicitly specify the path to save and load + the blobs enabling model caching feature. + + @param dir Path to the directory what will be used as cache. + @return reference to this parameter structure. + */ + GAPI_WRAP + OpenVINO& cfgCacheDir(const std::string &dir) { + cache_dir = dir; + return *this; + } + + /** @brief Specifies OpenVINO Execution Provider number of threads. + + This function is used to override the accelerator default value + of number of threads with this value at runtime. If this option + is not explicitly set, default value of 8 is used during build time. + + @param nthreads Number of threads. + @return reference to this parameter structure. + */ + GAPI_WRAP + OpenVINO& cfgNumThreads(size_t nthreads) { + num_of_threads = cv::util::make_optional(nthreads); + return *this; + } + + /** @brief Enables OpenVINO Execution Provider opencl throttling. + + This function is used to enable OpenCL queue throttling for GPU devices + (reduces CPU utilization when using GPU). + + @return reference to this parameter structure. + */ + GAPI_WRAP + OpenVINO& cfgEnableOpenCLThrottling() { + enable_opencl_throttling = true; + return *this; + } + + /** @brief Enables OpenVINO Execution Provider dynamic shapes. + + This function is used to enable OpenCL queue throttling for GPU devices + (reduces CPU utilization when using GPU). + This function is used to enable work with dynamic shaped models + whose shape will be set dynamically based on the infer input + image/data shape at run time in CPU. + + @return reference to this parameter structure. + */ + GAPI_WRAP + OpenVINO& cfgEnableDynamicShapes() { + enable_dynamic_shapes = true; + return *this; + } + + std::string device_id; + std::string cache_dir; + cv::optional device_type; + cv::optional num_of_threads; + bool enable_opencl_throttling = false; + bool enable_dynamic_shapes = false; +}; + +using EP = cv::util::variant; + +} // namespace ep + GAPI_EXPORTS cv::gapi::GBackend backend(); enum class TraitAs: int { @@ -78,6 +190,9 @@ struct ParamDesc { // when the generic infer parameters are unpacked (see GONNXBackendImpl::unpackKernel) std::unordered_map > generic_mstd; std::unordered_map generic_norm; + + cv::gapi::onnx::ep::EP execution_provider; + bool disable_mem_pattern; }; } // namespace detail @@ -115,6 +230,7 @@ public: desc.num_in = std::tuple_size::value; desc.num_out = std::tuple_size::value; desc.is_generic = false; + desc.disable_mem_pattern = false; }; /** @brief Specifies sequence of network input layers names for inference. @@ -279,6 +395,29 @@ public: return *this; } + /** @brief Specifies execution provider for runtime. + + The function is used to set ONNX Runtime OpenVINO Execution Provider options. + + @param ovep OpenVINO Execution Provider options. + @see cv::gapi::onnx::ep::OpenVINO. + + @return the reference on modified object. + */ + Params& cfgExecutionProvider(ep::OpenVINO&& ovep) { + desc.execution_provider = std::move(ovep); + return *this; + } + + /** @brief Disables the memory pattern optimization. + + @return the reference on modified object. + */ + Params& cfgDisableMemPattern() { + desc.disable_mem_pattern = true; + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::onnx::backend(); } std::string tag() const { return Net::tag(); } @@ -306,7 +445,7 @@ public: @param model_path path to model file (.onnx file). */ Params(const std::string& tag, const std::string& model_path) - : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {} }, m_tag(tag) {} + : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {}, {}, false }, m_tag(tag) {} void cfgMeanStdDev(const std::string &layer, const cv::Scalar &m, @@ -318,6 +457,14 @@ public: desc.generic_norm[layer] = flag; } + void cfgExecutionProvider(ep::OpenVINO&& ov_ep) { + desc.execution_provider = std::move(ov_ep); + } + + void cfgDisableMemPattern() { + desc.disable_mem_pattern = true; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::onnx::backend(); } std::string tag() const { return m_tag; } diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index 70698ffd48..bdd0f0232f 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -29,6 +29,7 @@ using map_string_and_string = std::map; using map_string_and_vector_size_t = std::map>; using map_string_and_vector_float = std::map>; using map_int_and_double = std::map; +using ep_OpenVINO = cv::gapi::onnx::ep::OpenVINO; // NB: Python wrapper generate T_U for T // This behavior is only observed for inputs diff --git a/modules/gapi/src/backends/onnx/bindings_onnx.cpp b/modules/gapi/src/backends/onnx/bindings_onnx.cpp index c9c5fc58fa..ada90bd130 100644 --- a/modules/gapi/src/backends/onnx/bindings_onnx.cpp +++ b/modules/gapi/src/backends/onnx/bindings_onnx.cpp @@ -21,6 +21,18 @@ cv::gapi::onnx::PyParams& cv::gapi::onnx::PyParams::cfgNormalize(const std::stri return *this; } +cv::gapi::onnx::PyParams& +cv::gapi::onnx::PyParams::cfgExecutionProvider(cv::gapi::onnx::ep::OpenVINO ov_ep) { + m_priv->cfgExecutionProvider(std::move(ov_ep)); + return *this; +} + +cv::gapi::onnx::PyParams& +cv::gapi::onnx::PyParams::cfgDisableMemPattern() { + m_priv->cfgDisableMemPattern(); + return *this; +} + cv::gapi::GBackend cv::gapi::onnx::PyParams::backend() const { return m_priv->backend(); } diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.cpp b/modules/gapi/src/backends/onnx/gonnxbackend.cpp index 1194caeeb3..79bdad93d9 100644 --- a/modules/gapi/src/backends/onnx/gonnxbackend.cpp +++ b/modules/gapi/src/backends/onnx/gonnxbackend.cpp @@ -143,6 +143,41 @@ public: void run(); }; +static void appendExecutionProvider(Ort::SessionOptions *session_options, + const cv::gapi::onnx::ep::EP &execution_provider) { + namespace ep = cv::gapi::onnx::ep; + switch (execution_provider.index()) { + case ep::EP::index_of(): { + GAPI_LOG_INFO(NULL, "OpenVINO Execution Provider is selected."); + const auto &ovep = cv::util::get(execution_provider); + OrtOpenVINOProviderOptions options; + options.device_id = ovep.device_id.c_str(); + options.cache_dir = ovep.cache_dir.c_str(); + options.enable_opencl_throttling = ovep.enable_opencl_throttling; + options.enable_dynamic_shapes = ovep.enable_dynamic_shapes; + // NB: If are not specified, will be taken from onnxruntime build. + if (ovep.device_type) { + options.device_type = ovep.device_type->c_str(); + } + if (ovep.num_of_threads) { + options.num_of_threads = *ovep.num_of_threads; + } + try { + session_options->AppendExecutionProvider_OpenVINO(options); + } catch (const std::exception &e) { + std::stringstream ss; + ss << "ONNX Backend: Failed to enable OpenVINO Execution Provider: " + << e.what() << "\nMake sure that onnxruntime has" + " been compiled with OpenVINO support."; + cv::util::throw_error(std::runtime_error(ss.str())); + } + break; + } + default: + break; + } +} + } // namespace onnx } // namespace gimpl } // namespace cv @@ -592,9 +627,13 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp) cv::util::throw_error(std::logic_error("Please specify output layer names for " + params.model_path)); } - // Create and initialize the ONNX session Ort::SessionOptions session_options; + cv::gimpl::onnx::appendExecutionProvider(&session_options, pp.execution_provider); + + if (pp.disable_mem_pattern) { + session_options.DisableMemPattern(); + } this_env = Ort::Env(ORT_LOGGING_LEVEL_WARNING, ""); #ifndef _WIN32 this_session = Ort::Session(this_env, params.model_path.data(), session_options); From e3cb5f80e73c8292f8191cc2ebb88eee25856406 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Fri, 21 Jul 2023 12:44:56 +0300 Subject: [PATCH 091/105] Merge pull request #24028 from VadimLevin:dev/vlevin/fix-flann-python-bindings Fix FLANN python bindings #24028 As a side-effect this patch improves reporting errors by FLANN `get_param`. resolves #21642 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/flann/include/opencv2/flann/any.h | 29 ++++- modules/flann/include/opencv2/flann/general.h | 2 + modules/flann/include/opencv2/flann/matrix.h | 3 + modules/flann/include/opencv2/flann/params.h | 18 ++- .../flann/include/opencv2/flann/result_set.h | 3 + modules/flann/misc/python/pyopencv_flann.hpp | 108 ++++++++++++------ .../python/test/test_flann_based_matcher.py | 30 +++++ 7 files changed, 148 insertions(+), 45 deletions(-) create mode 100644 modules/flann/misc/python/test/test_flann_based_matcher.py diff --git a/modules/flann/include/opencv2/flann/any.h b/modules/flann/include/opencv2/flann/any.h index 4906fec081..2228bd1cfc 100644 --- a/modules/flann/include/opencv2/flann/any.h +++ b/modules/flann/include/opencv2/flann/any.h @@ -19,16 +19,39 @@ #include #include +#include "opencv2/core/cvdef.h" +#include "opencv2/core/utility.hpp" + namespace cvflann { namespace anyimpl { -struct bad_any_cast +struct bad_any_cast : public std::exception { + bad_any_cast() = default; + + bad_any_cast(const char* src, const char* dst) + : message_(cv::format("cvflann::bad_any_cast(from %s to %s)", src, dst)) {} + + + const char* what() const noexcept override + { + return message_.c_str(); + } + +private: + std::string message_{"cvflann::bad_any_cast"}; }; +#ifndef CV_THROW_IF_TYPE_MISMATCH +#define CV_THROW_IF_TYPE_MISMATCH(src_type_info, dst_type_info) \ + if ((src_type_info) != (dst_type_info)) \ + throw cvflann::anyimpl::bad_any_cast((src_type_info).name(), \ + (dst_type_info).name()) +#endif + struct empty_any { }; @@ -271,7 +294,7 @@ public: template T& cast() { - if (policy->type() != typeid(T)) throw anyimpl::bad_any_cast(); + CV_THROW_IF_TYPE_MISMATCH(policy->type(), typeid(T)); T* r = reinterpret_cast(policy->get_value(&object)); return *r; } @@ -280,7 +303,7 @@ public: template const T& cast() const { - if (policy->type() != typeid(T)) throw anyimpl::bad_any_cast(); + CV_THROW_IF_TYPE_MISMATCH(policy->type(), typeid(T)); const T* r = reinterpret_cast(policy->get_value(&object)); return *r; } diff --git a/modules/flann/include/opencv2/flann/general.h b/modules/flann/include/opencv2/flann/general.h index 29fa8be121..e65cba2f8a 100644 --- a/modules/flann/include/opencv2/flann/general.h +++ b/modules/flann/include/opencv2/flann/general.h @@ -31,6 +31,8 @@ #ifndef OPENCV_FLANN_GENERAL_H_ #define OPENCV_FLANN_GENERAL_H_ +#include "opencv2/core/version.hpp" + #if CV_VERSION_MAJOR <= 4 //! @cond IGNORED diff --git a/modules/flann/include/opencv2/flann/matrix.h b/modules/flann/include/opencv2/flann/matrix.h index fb871bd73c..bfbf91ef5c 100644 --- a/modules/flann/include/opencv2/flann/matrix.h +++ b/modules/flann/include/opencv2/flann/matrix.h @@ -35,6 +35,9 @@ #include +#include "opencv2/core/cvdef.h" +#include "opencv2/flann/defines.h" + namespace cvflann { diff --git a/modules/flann/include/opencv2/flann/params.h b/modules/flann/include/opencv2/flann/params.h index c9093cde8c..1a8e127035 100644 --- a/modules/flann/include/opencv2/flann/params.h +++ b/modules/flann/include/opencv2/flann/params.h @@ -72,11 +72,16 @@ struct SearchParams : public IndexParams template -T get_param(const IndexParams& params, cv::String name, const T& default_value) +T get_param(const IndexParams& params, const cv::String& name, const T& default_value) { IndexParams::const_iterator it = params.find(name); if (it != params.end()) { - return it->second.cast(); + try { + return it->second.cast(); + } catch (const std::exception& e) { + CV_Error_(cv::Error::StsBadArg, + ("FLANN '%s' param type mismatch: %s", name.c_str(), e.what())); + } } else { return default_value; @@ -84,11 +89,16 @@ T get_param(const IndexParams& params, cv::String name, const T& default_value) } template -T get_param(const IndexParams& params, cv::String name) +T get_param(const IndexParams& params, const cv::String& name) { IndexParams::const_iterator it = params.find(name); if (it != params.end()) { - return it->second.cast(); + try { + return it->second.cast(); + } catch (const std::exception& e) { + CV_Error_(cv::Error::StsBadArg, + ("FLANN '%s' param type mismatch: %s", name.c_str(), e.what())); + } } else { FLANN_THROW(cv::Error::StsBadArg, cv::String("Missing parameter '")+name+cv::String("' in the parameters given")); diff --git a/modules/flann/include/opencv2/flann/result_set.h b/modules/flann/include/opencv2/flann/result_set.h index 47ad231105..c5d31e8ade 100644 --- a/modules/flann/include/opencv2/flann/result_set.h +++ b/modules/flann/include/opencv2/flann/result_set.h @@ -40,6 +40,9 @@ #include #include +#include "opencv2/core/base.hpp" +#include "opencv2/core/cvdef.h" + namespace cvflann { diff --git a/modules/flann/misc/python/pyopencv_flann.hpp b/modules/flann/misc/python/pyopencv_flann.hpp index 3d97edbb59..086ca5f09f 100644 --- a/modules/flann/misc/python/pyopencv_flann.hpp +++ b/modules/flann/misc/python/pyopencv_flann.hpp @@ -17,57 +17,89 @@ PyObject* pyopencv_from(const cvflann_flann_distance_t& value) template<> bool pyopencv_to(PyObject *o, cv::flann::IndexParams& p, const ArgInfo& info) { - CV_UNUSED(info); - bool ok = true; - PyObject* key = NULL; - PyObject* item = NULL; - Py_ssize_t pos = 0; - if (!o || o == Py_None) + { return true; + } - if(PyDict_Check(o)) { - while(PyDict_Next(o, &pos, &key, &item)) + if(!PyDict_Check(o)) + { + failmsg("Argument '%s' is not a dictionary", info.name); + return false; + } + + PyObject* key_obj = NULL; + PyObject* value_obj = NULL; + Py_ssize_t key_pos = 0; + + while(PyDict_Next(o, &key_pos, &key_obj, &value_obj)) + { + // get key + std::string key; + if (!getUnicodeString(key_obj, key)) { - // get key - std::string k; - if (!getUnicodeString(key, k)) + failmsg("Key at pos %lld is not a string", static_cast(key_pos)); + return false; + } + // key_arg_info.name is bound to key lifetime + const ArgInfo key_arg_info(key.c_str(), false); + + // get value + if (isBool(value_obj)) + { + npy_bool npy_value = NPY_FALSE; + if (PyArray_BoolConverter(value_obj, &npy_value) >= 0) { - ok = false; - break; + p.setBool(key, npy_value == NPY_TRUE); + continue; } - // get value - if( !!PyBool_Check(item) ) + PyErr_Clear(); + } + + int int_value = 0; + if (pyopencv_to(value_obj, int_value, key_arg_info)) + { + if (key == "algorithm") { - p.setBool(k, item == Py_True); - } - else if( PyInt_Check(item) ) - { - int value = (int)PyInt_AsLong(item); - if( strcmp(k.c_str(), "algorithm") == 0 ) - p.setAlgorithm(value); - else - p.setInt(k, value); - } - else if( PyFloat_Check(item) ) - { - double value = PyFloat_AsDouble(item); - p.setDouble(k, value); + p.setAlgorithm(int_value); } else { - std::string val_str; - if (!getUnicodeString(item, val_str)) - { - ok = false; - break; - } - p.setString(k, val_str); + p.setInt(key, int_value); } + continue; } - } + PyErr_Clear(); - return ok && !PyErr_Occurred(); + double flt_value = 0.0; + if (pyopencv_to(value_obj, flt_value, key_arg_info)) + { + if (key == "eps") + { + p.setFloat(key, static_cast(flt_value)); + } + else + { + p.setDouble(key, flt_value); + } + continue; + } + PyErr_Clear(); + + std::string str_value; + if (getUnicodeString(value_obj, str_value)) + { + p.setString(key, str_value); + continue; + } + PyErr_Clear(); + // All conversions are failed + failmsg("Failed to parse IndexParam with key '%s'. " + "Supported types: [bool, int, float, str]", key.c_str()); + return false; + + } + return true; } template<> diff --git a/modules/flann/misc/python/test/test_flann_based_matcher.py b/modules/flann/misc/python/test/test_flann_based_matcher.py new file mode 100644 index 0000000000..cf8c2ededd --- /dev/null +++ b/modules/flann/misc/python/test/test_flann_based_matcher.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# Python 2/3 compatibility +from __future__ import print_function + +import cv2 +import numpy as np + +from tests_common import NewOpenCVTests + + +class FlannBasedMatcher(NewOpenCVTests): + def test_all_parameters_can_be_passed(self): + img1 = self.get_sample("samples/data/right01.jpg") + img2 = self.get_sample("samples/data/right02.jpg") + + orb = cv2.ORB.create() + + kp1, des1 = orb.detectAndCompute(img1, None) + kp2, des2 = orb.detectAndCompute(img2, None) + FLANN_INDEX_KDTREE = 1 + index_param = dict(algorithm=FLANN_INDEX_KDTREE, trees=4) + search_param = dict(checks=32, sorted=True, eps=0.5, + explore_all_trees=False) + matcher = cv2.FlannBasedMatcher(index_param, search_param) + matches = matcher.knnMatch(np.float32(des1), np.float32(des2), k=2) + self.assertGreater(len(matches), 0) + + +if __name__ == '__main__': + NewOpenCVTests.bootstrap() From d96ff496b472fdfc44c6e7138ab665bf2bf4f6a6 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Fri, 21 Jul 2023 13:48:15 +0300 Subject: [PATCH 092/105] Increase eps for Test_Torch_nets.FastNeuralStyle_accuracy to prevent sporadic test failres with CUDA. --- modules/dnn/test/test_torch_importer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dnn/test/test_torch_importer.cpp b/modules/dnn/test/test_torch_importer.cpp index 8510ec4e64..ae39d5d22e 100644 --- a/modules/dnn/test/test_torch_importer.cpp +++ b/modules/dnn/test/test_torch_importer.cpp @@ -566,14 +566,14 @@ TEST_P(Test_Torch_nets, FastNeuralStyle_accuracy) } else if(target == DNN_TARGET_CUDA_FP16) { - normAssert(out, refBlob, "", 0.6, 25); + normAssert(out, refBlob, "", 0.6, 26); } else if (target == DNN_TARGET_CPU_FP16) { normAssert(out, refBlob, "", 0.62, 25); } else - normAssert(out, refBlob, "", 0.5, 1.1); + normAssert(out, refBlob, "", 0.5, 1.11); } } From ec7689421d7006f25d145da148be0141d0afc8a9 Mon Sep 17 00:00:00 2001 From: zixgo <1145922655@qq.com> Date: Fri, 21 Jul 2023 19:54:43 +0800 Subject: [PATCH 093/105] fix compilation error on Windows ARM, use vaddq_f32 instead of += --- .../dnn/src/layers/cpu_kernels/conv_block.simd.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp b/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp index 27b0d4ba1f..3310a91b6b 100644 --- a/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp +++ b/modules/dnn/src/layers/cpu_kernels/conv_block.simd.hpp @@ -452,13 +452,13 @@ void convBlockMR1_F32(int np, const float * a, const float * b, float *c, const if (init_c) { - c0 += vld1q_f32(c); - c1 += vld1q_f32(c + 4); - c2 += vld1q_f32(c + 8); - c3 += vld1q_f32(c + 12); - c4 += vld1q_f32(c + 16); - c5 += vld1q_f32(c + 20); - c6 += vld1q_f32(c + 24); + c0 = vaddq_f32(c0, vld1q_f32(c)); + c1 = vaddq_f32(c1, vld1q_f32(c + 4)); + c2 = vaddq_f32(c2, vld1q_f32(c + 8)); + c3 = vaddq_f32(c3, vld1q_f32(c + 12)); + c4 = vaddq_f32(c4, vld1q_f32(c + 16)); + c5 = vaddq_f32(c5, vld1q_f32(c + 20)); + c6 = vaddq_f32(c6, vld1q_f32(c + 24)); } if (ifMinMaxAct) From 2fc7d219714fc7dee50446b47cac5207ab474484 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Fri, 21 Jul 2023 14:57:32 +0300 Subject: [PATCH 094/105] Merge pull request #24029 from VadimLevin:dev/vlevin/python-add-cuda-stream-to-simple-types feat: add cuda_Stream and cuda_GpuMat to simple types mapping #24029 This patch fixes usage of `cuda::Stream` in function arguments. Affected modules: `cudacodec`: [`using namespace cuda`](https://github.com/opencv/opencv_contrib/blob/9dfe233020f669f17021dfc456fe77531e776b74/modules/cudacodec/include/opencv2/cudacodec.hpp#L62) in public `cudacodec.hpp` header can be removed after merge of the patch. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/python/src2/gen2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index 0ba643f12b..fdcae20187 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -231,6 +231,8 @@ simple_argtype_mapping = { "c_string": ArgTypeInfo("char*", FormatStrings.string, '(char*)""'), "string": ArgTypeInfo("std::string", FormatStrings.object, None, True), "Stream": ArgTypeInfo("Stream", FormatStrings.object, 'Stream::Null()', True), + "cuda_Stream": ArgTypeInfo("cuda::Stream", FormatStrings.object, "cuda::Stream::Null()", True), + "cuda_GpuMat": ArgTypeInfo("cuda::GpuMat", FormatStrings.object, "cuda::GpuMat()", True), "UMat": ArgTypeInfo("UMat", FormatStrings.object, 'UMat()', True), # FIXIT: switch to CV_EXPORTS_W_SIMPLE as UMat is already a some kind of smart pointer } From 0bcd66d553236936b6471ce879b9664300a332c8 Mon Sep 17 00:00:00 2001 From: "Ivashechkin, Maxim (PG/R - Comp Sci & Elec Eng)" Date: Sat, 22 Jul 2023 10:44:37 +0100 Subject: [PATCH 095/105] remove unused --- modules/calib3d/src/usac.hpp | 14 +---- modules/calib3d/src/usac/utils.cpp | 90 ------------------------------ 2 files changed, 2 insertions(+), 102 deletions(-) diff --git a/modules/calib3d/src/usac.hpp b/modules/calib3d/src/usac.hpp index 85b2730e4a..57d91ab7f3 100644 --- a/modules/calib3d/src/usac.hpp +++ b/modules/calib3d/src/usac.hpp @@ -176,7 +176,7 @@ public: //-------------------------- ESSENTIAL MATRIX ----------------------- class EssentialNonMinimalSolverViaF : public NonMinimalSolver { public: -static Ptr create(const Mat &points_, const cv::Mat &K1, const Mat &K2); + static Ptr create(const Mat &points_, const cv::Mat &K1, const Mat &K2); }; class EssentialNonMinimalSolverViaT : public NonMinimalSolver { @@ -432,7 +432,7 @@ public: }; class EssentialEstimator : public Estimator { -public : +public: static Ptr create (const Ptr &min_solver_, const Ptr &non_min_solver_, const Ptr °eneracy_); }; @@ -542,11 +542,6 @@ public: int cell_size_x_img1_, int cell_size_y_img1_, int cell_size_x_img2_, int cell_size_y_img2_, int max_neighbors); }; -class GridNeighborhoodGraph2Images : public NeighborhoodGraph { -public: - static Ptr create(const Mat &points, int points_size, - float cell_size_x_img1_, float cell_size_y_img1_, float cell_size_x_img2_, float cell_size_y_img2_); -}; ////////////////////////////////////// UNIFORM SAMPLER //////////////////////////////////////////// class UniformSampler : public Sampler { @@ -554,11 +549,6 @@ public: static Ptr create(int state, int sample_size_, int points_size_); }; -class QuasiUniformSampler : public Sampler { -public: - static Ptr create(int state, int sample_size_, int points_size_); -}; - /////////////////////////////////// PROSAC (SIMPLE) SAMPLER /////////////////////////////////////// class ProsacSimpleSampler : public Sampler { public: diff --git a/modules/calib3d/src/usac/utils.cpp b/modules/calib3d/src/usac/utils.cpp index da1956f26c..204b31cf92 100644 --- a/modules/calib3d/src/usac/utils.cpp +++ b/modules/calib3d/src/usac/utils.cpp @@ -826,94 +826,4 @@ Ptr GridNeighborhoodGraph::create(const Mat &points, return makePtr(points, points_size, cell_size_x_img1_, cell_size_y_img1_, cell_size_x_img2_, cell_size_y_img2_, max_neighbors); } - -class GridNeighborhoodGraph2ImagesImpl : public GridNeighborhoodGraph2Images { -private: - // This struct is used for the nearest neighbors search by griding two images. - struct CellCoord { - int c1x, c1y; - CellCoord (int c1x_, int c1y_) { - c1x = c1x_; c1y = c1y_; - } - bool operator==(const CellCoord &o) const { - return c1x == o.c1x && c1y == o.c1y; - } - bool operator<(const CellCoord &o) const { - if (c1x < o.c1x) return true; - return c1x == o.c1x && c1y < o.c1y; - } - }; - - std::vector> graph; -public: - GridNeighborhoodGraph2ImagesImpl (const Mat &container_, int points_size, - float cell_size_x_img1, float cell_size_y_img1, float cell_size_x_img2, float cell_size_y_img2) { - - std::map> neighbors_map1, neighbors_map2; - const auto * const container = (float *) container_.data; - // Key is cell position. The value is indexes of neighbors. - - const auto cell_sz_x1 = 1.f / cell_size_x_img1, - cell_sz_y1 = 1.f / cell_size_y_img1, - cell_sz_x2 = 1.f / cell_size_x_img2, - cell_sz_y2 = 1.f / cell_size_y_img2; - const int dimension = container_.cols; - for (int i = 0; i < points_size; i++) { - const int idx = dimension * i; - neighbors_map1[CellCoord((int)(container[idx ] * cell_sz_x1), - (int)(container[idx+1] * cell_sz_y1))].emplace_back(i); - neighbors_map2[CellCoord((int)(container[idx+2] * cell_sz_x2), - (int)(container[idx+3] * cell_sz_y2))].emplace_back(i); - } - - //--------- create a graph ---------- - graph = std::vector>(points_size); - - // store neighbors cells into graph (2D vector) - for (const auto &cell : neighbors_map1) { - const int neighbors_in_cell = static_cast(cell.second.size()); - // only one point in cell -> no neighbors - if (neighbors_in_cell < 2) continue; - - const std::vector &neighbors = cell.second; - // ---------- fill graph ----- - // for speed-up we make no symmetric graph, eg, x has a neighbor y, but y does not have x - const int v_in_cell = neighbors[0]; - // there is always at least one neighbor - auto &graph_row = graph[v_in_cell]; - graph_row.reserve(neighbors_in_cell); - for (int n : neighbors) - if (n != v_in_cell) - graph_row.emplace_back(n); - } - - // fill neighbors of a second image - for (const auto &cell : neighbors_map2) { - if (cell.second.size() < 2) continue; - const std::vector &neighbors = cell.second; - const int v_in_cell = neighbors[0]; - auto &graph_row = graph[v_in_cell]; - for (const int &n : neighbors) - if (n != v_in_cell) { - bool has = false; - for (const int &nn : graph_row) - if (n == nn) { - has = true; break; - } - if (!has) graph_row.emplace_back(n); - } - } - } - const std::vector> &getGraph () const override { return graph; } - inline const std::vector &getNeighbors(int point_idx) const override { - // Note, neighbors vector also includes point_idx! - return graph[point_idx]; - } -}; - -Ptr GridNeighborhoodGraph2Images::create(const Mat &points, - int points_size, float cell_size_x_img1_, float cell_size_y_img1_, float cell_size_x_img2_, float cell_size_y_img2_) { - return makePtr(points, points_size, - cell_size_x_img1_, cell_size_y_img1_, cell_size_x_img2_, cell_size_y_img2_); -} }} \ No newline at end of file From a5ba18c20e46669a6baaf45e000f7ff775a3a3cc Mon Sep 17 00:00:00 2001 From: Stanley Mwangi Date: Tue, 25 Jul 2023 11:21:13 +0300 Subject: [PATCH 096/105] popcnt is not a windows ARM intrinsic. --- 3rdparty/openjpeg/openjp2/ht_dec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/openjpeg/openjp2/ht_dec.c b/3rdparty/openjpeg/openjp2/ht_dec.c index 1eb4d525f1..e2f3afd6a3 100644 --- a/3rdparty/openjpeg/openjp2/ht_dec.c +++ b/3rdparty/openjpeg/openjp2/ht_dec.c @@ -69,7 +69,7 @@ static OPJ_BOOL only_cleanup_pass_is_decoded = OPJ_FALSE; static INLINE OPJ_UINT32 population_count(OPJ_UINT32 val) { -#ifdef OPJ_COMPILER_MSVC +#if defined(OPJ_COMPILER_MSVC) && (defined(_M_IX86) || defined(_M_AMD64)) return (OPJ_UINT32)__popcnt(val); #elif (defined OPJ_COMPILER_GNUC) return (OPJ_UINT32)__builtin_popcount(val); From b22c2505a8833c52f84f181a4ce414f93b5c1cd7 Mon Sep 17 00:00:00 2001 From: Alexander Smorkalov Date: Tue, 25 Jul 2023 14:50:55 +0300 Subject: [PATCH 097/105] Disable warning C5054 in VS 2022 C++20 --- modules/core/include/opencv2/core/mat.inl.hpp | 2 +- modules/imgproc/include/opencv2/imgproc/imgproc_c.h | 8 ++++++++ modules/videoio/include/opencv2/videoio.hpp | 7 +++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/modules/core/include/opencv2/core/mat.inl.hpp b/modules/core/include/opencv2/core/mat.inl.hpp index 886b82c6a0..f0eed783a5 100644 --- a/modules/core/include/opencv2/core/mat.inl.hpp +++ b/modules/core/include/opencv2/core/mat.inl.hpp @@ -51,7 +51,7 @@ #ifdef _MSC_VER #pragma warning( push ) -#pragma warning( disable: 4127 ) +#pragma warning( disable: 4127 5054 ) #endif #if defined(CV_SKIP_DISABLE_CLANG_ENUM_WARNINGS) diff --git a/modules/imgproc/include/opencv2/imgproc/imgproc_c.h b/modules/imgproc/include/opencv2/imgproc/imgproc_c.h index 86dc119fdd..e97b802e69 100644 --- a/modules/imgproc/include/opencv2/imgproc/imgproc_c.h +++ b/modules/imgproc/include/opencv2/imgproc/imgproc_c.h @@ -209,6 +209,10 @@ CVAPI(void) cvCvtColor( const CvArr* src, CvArr* dst, int code ); CVAPI(void) cvResize( const CvArr* src, CvArr* dst, int interpolation CV_DEFAULT( CV_INTER_LINEAR )); +#ifdef _MSC_VER +#pragma warning( push ) +#pragma warning( disable: 5054 ) +#endif /** @brief Warps image with affine transform @note ::cvGetQuadrangleSubPix is similar to ::cvWarpAffine, but the outliers are extrapolated using replication border mode. @@ -273,6 +277,10 @@ CVAPI(void) cvLinearPolar( const CvArr* src, CvArr* dst, CvPoint2D32f center, double maxRadius, int flags CV_DEFAULT(CV_INTER_LINEAR+CV_WARP_FILL_OUTLIERS)); +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + /** @brief Returns a structuring element of the specified size and shape for morphological operations. @note the created structuring element IplConvKernel\* element must be released in the end using diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 1c3f0f5eb0..dbed243b56 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -311,6 +311,10 @@ enum { CAP_PROP_OPENNI_OUTPUT_MODE = 100, CAP_PROP_OPENNI2_MIRROR = 111 }; +#ifdef _MSC_VER +#pragma warning( push ) +#pragma warning( disable: 5054 ) +#endif //! OpenNI shortcuts enum { CAP_OPENNI_IMAGE_GENERATOR_PRESENT = CAP_OPENNI_IMAGE_GENERATOR + CAP_PROP_OPENNI_GENERATOR_PRESENT, CAP_OPENNI_IMAGE_GENERATOR_OUTPUT_MODE = CAP_OPENNI_IMAGE_GENERATOR + CAP_PROP_OPENNI_OUTPUT_MODE, @@ -321,6 +325,9 @@ enum { CAP_OPENNI_IMAGE_GENERATOR_PRESENT = CAP_OPENNI_IMAGE_GENERATOR + CAP_OPENNI_DEPTH_GENERATOR_REGISTRATION_ON = CAP_OPENNI_DEPTH_GENERATOR_REGISTRATION, CAP_OPENNI_IR_GENERATOR_PRESENT = CAP_OPENNI_IR_GENERATOR + CAP_PROP_OPENNI_GENERATOR_PRESENT, }; +#ifdef _MSC_VER +#pragma warning( pop ) +#endif //! OpenNI data given from depth generator enum { CAP_OPENNI_DEPTH_MAP = 0, //!< Depth values in mm (CV_16UC1) From be29c99d5a9b70ed691a7bf6185a75a540687b7c Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Fri, 21 Jul 2023 17:50:48 +0300 Subject: [PATCH 098/105] feat: add cuda_GpuMat to big types This patch enables passing GpuMat as an in/out argument in several functions. --- modules/python/src2/gen2.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/python/src2/gen2.py b/modules/python/src2/gen2.py index fdcae20187..95bd0495cf 100755 --- a/modules/python/src2/gen2.py +++ b/modules/python/src2/gen2.py @@ -511,14 +511,17 @@ class ArgInfo(object): return self.enclosing_arg.name + '.' + self.name def isbig(self): - return self.tp in ["Mat", "vector_Mat", "cuda::GpuMat", "GpuMat", "vector_GpuMat", "UMat", "vector_UMat"] # or self.tp.startswith("vector") + return self.tp in ["Mat", "vector_Mat", + "cuda::GpuMat", "cuda_GpuMat", "GpuMat", + "vector_GpuMat", "vector_cuda_GpuMat", + "UMat", "vector_UMat"] # or self.tp.startswith("vector") def crepr(self): return "ArgInfo(\"%s\", %d)" % (self.name, self.outputarg) def find_argument_class_info(argument_type, function_namespace, - function_class_name, known_classes): + function_class_name, known_classes): # type: (str, str, str, dict[str, ClassInfo]) -> ClassInfo | None """Tries to find corresponding class info for the provided argument type From a817813b5090808925bc474a31b4d61e43cafc85 Mon Sep 17 00:00:00 2001 From: Anatoliy Talamanov Date: Wed, 26 Jul 2023 14:00:20 +0100 Subject: [PATCH 099/105] Merge pull request #24045 from TolyaTalamanov:at/add-onnx-directml-execution-provider G-API: Support DirectML Execution Provider for ONNXRT Backend #24045 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [ ] I agree to contribute to the project under Apache 2 License. - [ ] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [ ] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [ ] The feature is well documented and sample code can be built with the project CMake --- cmake/FindONNX.cmake | 15 +++ modules/gapi/CMakeLists.txt | 4 + .../opencv2/gapi/infer/bindings_onnx.hpp | 5 +- .../gapi/include/opencv2/gapi/infer/onnx.hpp | 99 ++++++++++++------- modules/gapi/misc/python/pyopencv_gapi.hpp | 1 + .../gapi/src/backends/onnx/bindings_onnx.cpp | 10 +- modules/gapi/src/backends/onnx/dml_ep.cpp | 40 ++++++++ modules/gapi/src/backends/onnx/dml_ep.hpp | 23 +++++ .../gapi/src/backends/onnx/gonnxbackend.cpp | 64 +++++++----- 9 files changed, 198 insertions(+), 63 deletions(-) create mode 100644 modules/gapi/src/backends/onnx/dml_ep.cpp create mode 100644 modules/gapi/src/backends/onnx/dml_ep.hpp diff --git a/cmake/FindONNX.cmake b/cmake/FindONNX.cmake index 56dd6d5098..b2c79a9031 100644 --- a/cmake/FindONNX.cmake +++ b/cmake/FindONNX.cmake @@ -16,7 +16,22 @@ if(ONNXRT_ROOT_DIR) CMAKE_FIND_ROOT_PATH_BOTH) endif() +macro(detect_onxxrt_ep filename dir have_ep_var) + find_path(ORT_EP_INCLUDE ${filename} ${dir} CMAKE_FIND_ROOT_PATH_BOTH) + if(ORT_EP_INCLUDE) + set(${have_ep_var} TRUE) + endif() +endmacro() + if(ORT_LIB AND ORT_INCLUDE) + # Check DirectML Execution Provider availability + get_filename_component(dml_dir ${ONNXRT_ROOT_DIR}/include/onnxruntime/core/providers/dml ABSOLUTE) + detect_onxxrt_ep( + dml_provider_factory.h + ${dml_dir} + HAVE_ONNX_DML + ) + set(HAVE_ONNX TRUE) # For CMake output only set(ONNX_LIBRARIES "${ORT_LIB}" CACHE STRING "ONNX Runtime libraries") diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 31322d533a..e30cb77e9e 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -162,6 +162,7 @@ set(gapi_srcs # ONNX backend src/backends/onnx/gonnxbackend.cpp + src/backends/onnx/dml_ep.cpp # Render backend src/backends/render/grenderocv.cpp @@ -366,6 +367,9 @@ endif() if(HAVE_ONNX) ocv_target_link_libraries(${the_module} PRIVATE ${ONNX_LIBRARY}) ocv_target_compile_definitions(${the_module} PRIVATE HAVE_ONNX=1) + if(HAVE_ONNX_DML) + ocv_target_compile_definitions(${the_module} PRIVATE HAVE_ONNX_DML=1) + endif() if(TARGET opencv_test_gapi) ocv_target_compile_definitions(opencv_test_gapi PRIVATE HAVE_ONNX=1) ocv_target_link_libraries(opencv_test_gapi PRIVATE ${ONNX_LIBRARY}) diff --git a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp index 0f764fe2cd..4ba829df09 100644 --- a/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/bindings_onnx.hpp @@ -34,7 +34,10 @@ public: PyParams& cfgNormalize(const std::string &layer_name, bool flag); GAPI_WRAP - PyParams& cfgExecutionProvider(ep::OpenVINO ov_ep); + PyParams& cfgAddExecutionProvider(ep::OpenVINO ep); + + GAPI_WRAP + PyParams& cfgAddExecutionProvider(ep::DirectML ep); GAPI_WRAP PyParams& cfgDisableMemPattern(); diff --git a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp index 9c2118c3ad..64b855acd7 100644 --- a/modules/gapi/include/opencv2/gapi/infer/onnx.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/onnx.hpp @@ -45,28 +45,13 @@ struct GAPI_EXPORTS_W_SIMPLE OpenVINO { /** @brief Class constructor. - Constructs OpenVINO parameters based on device information. + Constructs OpenVINO parameters based on device type information. - @param device Target device to use. + @param dev_type Target device type to use. ("CPU_FP32", "GPU_FP16", etc) */ GAPI_WRAP - OpenVINO(const std::string &device) - : device_id(device) { - } - - /** @brief Specifies OpenVINO Execution Provider device type. - - This function is used to override the accelerator hardware type - and precision at runtime. If this option is not explicitly configured, default - hardware and precision specified during onnxruntime build time is used. - - @param type Device type ("CPU_FP32", "GPU_FP16", etc) - @return reference to this parameter structure. - */ - GAPI_WRAP - OpenVINO& cfgDeviceType(const std::string &type) { - device_type = cv::util::make_optional(type); - return *this; + explicit OpenVINO(const std::string &dev_type) + : device_type(dev_type) { } /** @brief Specifies OpenVINO Execution Provider cache dir. @@ -86,15 +71,14 @@ struct GAPI_EXPORTS_W_SIMPLE OpenVINO { /** @brief Specifies OpenVINO Execution Provider number of threads. This function is used to override the accelerator default value - of number of threads with this value at runtime. If this option - is not explicitly set, default value of 8 is used during build time. + of number of threads with this value at runtime. @param nthreads Number of threads. @return reference to this parameter structure. */ GAPI_WRAP OpenVINO& cfgNumThreads(size_t nthreads) { - num_of_threads = cv::util::make_optional(nthreads); + num_of_threads = nthreads; return *this; } @@ -127,15 +111,39 @@ struct GAPI_EXPORTS_W_SIMPLE OpenVINO { return *this; } - std::string device_id; + std::string device_type; std::string cache_dir; - cv::optional device_type; - cv::optional num_of_threads; + size_t num_of_threads = 0; bool enable_opencl_throttling = false; bool enable_dynamic_shapes = false; }; -using EP = cv::util::variant; +/** + * @brief This structure provides functions + * that fill inference options for ONNX DirectML Execution Provider. + * Please follow https://onnxruntime.ai/docs/execution-providers/DirectML-ExecutionProvider.html#directml-execution-provider + */ +class GAPI_EXPORTS_W_SIMPLE DirectML { +public: + // NB: Used from python. + /// @private -- Exclude this constructor from OpenCV documentation + GAPI_WRAP + DirectML() = default; + + /** @brief Class constructor. + + Constructs DirectML parameters based on device id. + + @param device_id Target device id to use. ("0", "1", etc) + */ + GAPI_WRAP + explicit DirectML(const int device_id) : ddesc(device_id) { }; + + using DeviceDesc = cv::util::variant; + DeviceDesc ddesc; +}; + +using EP = cv::util::variant; } // namespace ep @@ -191,7 +199,7 @@ struct ParamDesc { std::unordered_map > generic_mstd; std::unordered_map generic_norm; - cv::gapi::onnx::ep::EP execution_provider; + std::vector execution_providers; bool disable_mem_pattern; }; } // namespace detail @@ -395,17 +403,31 @@ public: return *this; } - /** @brief Specifies execution provider for runtime. + /** @brief Adds execution provider for runtime. - The function is used to set ONNX Runtime OpenVINO Execution Provider options. + The function is used to add ONNX Runtime OpenVINO Execution Provider options. - @param ovep OpenVINO Execution Provider options. + @param ep OpenVINO Execution Provider options. @see cv::gapi::onnx::ep::OpenVINO. @return the reference on modified object. */ - Params& cfgExecutionProvider(ep::OpenVINO&& ovep) { - desc.execution_provider = std::move(ovep); + Params& cfgAddExecutionProvider(ep::OpenVINO&& ep) { + desc.execution_providers.emplace_back(std::move(ep)); + return *this; + } + + /** @brief Adds execution provider for runtime. + + The function is used to add ONNX Runtime DirectML Execution Provider options. + + @param ep DirectML Execution Provider options. + @see cv::gapi::onnx::ep::DirectML. + + @return the reference on modified object. + */ + Params& cfgAddExecutionProvider(ep::DirectML&& ep) { + desc.execution_providers.emplace_back(std::move(ep)); return *this; } @@ -447,20 +469,29 @@ public: Params(const std::string& tag, const std::string& model_path) : desc{model_path, 0u, 0u, {}, {}, {}, {}, {}, {}, {}, {}, {}, true, {}, {}, {}, false }, m_tag(tag) {} + /** @see onnx::Params::cfgMeanStdDev. */ void cfgMeanStdDev(const std::string &layer, const cv::Scalar &m, const cv::Scalar &s) { desc.generic_mstd[layer] = std::make_pair(m, s); } + /** @see onnx::Params::cfgNormalize. */ void cfgNormalize(const std::string &layer, bool flag) { desc.generic_norm[layer] = flag; } - void cfgExecutionProvider(ep::OpenVINO&& ov_ep) { - desc.execution_provider = std::move(ov_ep); + /** @see onnx::Params::cfgAddExecutionProvider. */ + void cfgAddExecutionProvider(ep::OpenVINO&& ep) { + desc.execution_providers.emplace_back(std::move(ep)); } + /** @see onnx::Params::cfgAddExecutionProvider. */ + void cfgAddExecutionProvider(ep::DirectML&& ep) { + desc.execution_providers.emplace_back(std::move(ep)); + } + + /** @see onnx::Params::cfgDisableMemPattern. */ void cfgDisableMemPattern() { desc.disable_mem_pattern = true; } diff --git a/modules/gapi/misc/python/pyopencv_gapi.hpp b/modules/gapi/misc/python/pyopencv_gapi.hpp index bdd0f0232f..60d5f85479 100644 --- a/modules/gapi/misc/python/pyopencv_gapi.hpp +++ b/modules/gapi/misc/python/pyopencv_gapi.hpp @@ -30,6 +30,7 @@ using map_string_and_vector_size_t = std::map> using map_string_and_vector_float = std::map>; using map_int_and_double = std::map; using ep_OpenVINO = cv::gapi::onnx::ep::OpenVINO; +using ep_DirectML = cv::gapi::onnx::ep::DirectML; // NB: Python wrapper generate T_U for T // This behavior is only observed for inputs diff --git a/modules/gapi/src/backends/onnx/bindings_onnx.cpp b/modules/gapi/src/backends/onnx/bindings_onnx.cpp index ada90bd130..6051c6bb4d 100644 --- a/modules/gapi/src/backends/onnx/bindings_onnx.cpp +++ b/modules/gapi/src/backends/onnx/bindings_onnx.cpp @@ -22,8 +22,14 @@ cv::gapi::onnx::PyParams& cv::gapi::onnx::PyParams::cfgNormalize(const std::stri } cv::gapi::onnx::PyParams& -cv::gapi::onnx::PyParams::cfgExecutionProvider(cv::gapi::onnx::ep::OpenVINO ov_ep) { - m_priv->cfgExecutionProvider(std::move(ov_ep)); +cv::gapi::onnx::PyParams::cfgAddExecutionProvider(cv::gapi::onnx::ep::OpenVINO ep) { + m_priv->cfgAddExecutionProvider(std::move(ep)); + return *this; +} + +cv::gapi::onnx::PyParams& +cv::gapi::onnx::PyParams::cfgAddExecutionProvider(cv::gapi::onnx::ep::DirectML ep) { + m_priv->cfgAddExecutionProvider(std::move(ep)); return *this; } diff --git a/modules/gapi/src/backends/onnx/dml_ep.cpp b/modules/gapi/src/backends/onnx/dml_ep.cpp new file mode 100644 index 0000000000..7f59e1f3d6 --- /dev/null +++ b/modules/gapi/src/backends/onnx/dml_ep.cpp @@ -0,0 +1,40 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2023 Intel Corporation + +#include "backends/onnx/dml_ep.hpp" +#include "logger.hpp" + +#ifdef HAVE_ONNX +#include + +#ifdef HAVE_ONNX_DML +#include "../providers/dml/dml_provider_factory.h" + +void cv::gimpl::onnx::addDMLExecutionProvider(Ort::SessionOptions *session_options, + const cv::gapi::onnx::ep::DirectML &dml_ep) { + namespace ep = cv::gapi::onnx::ep; + GAPI_Assert(cv::util::holds_alternative(dml_ep.ddesc)); + const int device_id = cv::util::get(dml_ep.ddesc); + try { + OrtSessionOptionsAppendExecutionProvider_DML(*session_options, device_id); + } catch (const std::exception &e) { + std::stringstream ss; + ss << "ONNX Backend: Failed to enable DirectML" + << " Execution Provider: " << e.what(); + cv::util::throw_error(std::runtime_error(ss.str())); + } +} + +#else // HAVE_ONNX_DML + +void cv::gimpl::onnx::addDMLExecutionProvider(Ort::SessionOptions*, + const cv::gapi::onnx::ep::DirectML&) { + util::throw_error(std::runtime_error("G-API has been compiled with ONNXRT" + " without DirectML support")); +} + +#endif // HAVE_ONNX_DML +#endif // HAVE_ONNX diff --git a/modules/gapi/src/backends/onnx/dml_ep.hpp b/modules/gapi/src/backends/onnx/dml_ep.hpp new file mode 100644 index 0000000000..d7e43dc888 --- /dev/null +++ b/modules/gapi/src/backends/onnx/dml_ep.hpp @@ -0,0 +1,23 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2023 Intel Corporation + +#ifndef OPENCV_GAPI_DML_EP_HPP +#define OPENCV_GAPI_DML_EP_HPP + +#include "opencv2/gapi/infer/onnx.hpp" +#ifdef HAVE_ONNX + +#include + +namespace cv { +namespace gimpl { +namespace onnx { +void addDMLExecutionProvider(Ort::SessionOptions *session_options, + const cv::gapi::onnx::ep::DirectML &dml_ep); +}}} + +#endif // HAVE_ONNX +#endif // OPENCV_GAPI_DML_EP_HPP diff --git a/modules/gapi/src/backends/onnx/gonnxbackend.cpp b/modules/gapi/src/backends/onnx/gonnxbackend.cpp index 79bdad93d9..b90d4d6974 100644 --- a/modules/gapi/src/backends/onnx/gonnxbackend.cpp +++ b/modules/gapi/src/backends/onnx/gonnxbackend.cpp @@ -9,6 +9,8 @@ #ifdef HAVE_ONNX +#include "backends/onnx/dml_ep.hpp" + #include // any_of #include #include @@ -143,37 +145,44 @@ public: void run(); }; -static void appendExecutionProvider(Ort::SessionOptions *session_options, - const cv::gapi::onnx::ep::EP &execution_provider) { +static void addOpenVINOExecutionProvider(Ort::SessionOptions *session_options, + const cv::gapi::onnx::ep::OpenVINO &ov_ep) { + OrtOpenVINOProviderOptions options; + options.device_type = ov_ep.device_type.c_str(); + options.cache_dir = ov_ep.cache_dir.c_str(); + options.num_of_threads = ov_ep.num_of_threads; + options.enable_opencl_throttling = ov_ep.enable_opencl_throttling; + options.enable_dynamic_shapes = ov_ep.enable_dynamic_shapes; + options.context = nullptr; + + try { + session_options->AppendExecutionProvider_OpenVINO(options); + } catch (const std::exception &e) { + std::stringstream ss; + ss << "ONNX Backend: Failed to enable OpenVINO" + << " Execution Provider: " << e.what(); + cv::util::throw_error(std::runtime_error(ss.str())); + } +} + +static void addExecutionProvider(Ort::SessionOptions *session_options, + const cv::gapi::onnx::ep::EP &execution_provider) { namespace ep = cv::gapi::onnx::ep; switch (execution_provider.index()) { case ep::EP::index_of(): { - GAPI_LOG_INFO(NULL, "OpenVINO Execution Provider is selected."); - const auto &ovep = cv::util::get(execution_provider); - OrtOpenVINOProviderOptions options; - options.device_id = ovep.device_id.c_str(); - options.cache_dir = ovep.cache_dir.c_str(); - options.enable_opencl_throttling = ovep.enable_opencl_throttling; - options.enable_dynamic_shapes = ovep.enable_dynamic_shapes; - // NB: If are not specified, will be taken from onnxruntime build. - if (ovep.device_type) { - options.device_type = ovep.device_type->c_str(); - } - if (ovep.num_of_threads) { - options.num_of_threads = *ovep.num_of_threads; - } - try { - session_options->AppendExecutionProvider_OpenVINO(options); - } catch (const std::exception &e) { - std::stringstream ss; - ss << "ONNX Backend: Failed to enable OpenVINO Execution Provider: " - << e.what() << "\nMake sure that onnxruntime has" - " been compiled with OpenVINO support."; - cv::util::throw_error(std::runtime_error(ss.str())); - } + GAPI_LOG_INFO(NULL, "OpenVINO Execution Provider is added."); + const auto &ov_ep = cv::util::get(execution_provider); + addOpenVINOExecutionProvider(session_options, ov_ep); break; } + case ep::EP::index_of(): { + GAPI_LOG_INFO(NULL, "DirectML Execution Provider is added."); + const auto &dml_ep = cv::util::get(execution_provider); + addDMLExecutionProvider(session_options, dml_ep); + break; + } default: + GAPI_LOG_INFO(NULL, "CPU Execution Provider is added."); break; } } @@ -629,7 +638,10 @@ ONNXCompiled::ONNXCompiled(const gapi::onnx::detail::ParamDesc &pp) } // Create and initialize the ONNX session Ort::SessionOptions session_options; - cv::gimpl::onnx::appendExecutionProvider(&session_options, pp.execution_provider); + GAPI_LOG_INFO(NULL, "Adding Execution Providers for \"" << pp.model_path << "\""); + for (const auto &ep : pp.execution_providers) { + cv::gimpl::onnx::addExecutionProvider(&session_options, ep); + } if (pp.disable_mem_pattern) { session_options.DisableMemPattern(); From dad72fd47b4012f5f38e514c13cc0d5a32c87880 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Wed, 26 Jul 2023 16:34:43 +0300 Subject: [PATCH 100/105] feat: add highgui functions to typing stubs Manually add typing stubs for functions defined in `cv2_highgui.hpp`: - `createTrackbar` ```python def createTrackbar(trackbarName: str, windowName: str, value: int, count: int, onChange: Callable[[int], None]) -> None: ... ``` - `createButton` ```python def createButton(buttonName: str, onChange: Callable[[tuple[int] | tuple[int, Any]], None], userData: Any | None = ..., buttonType: int = ..., initialButtonState: int = ...) -> None: ... ``` - `setMouseCallback` ```python def setMouseCallback( windowName: str, onMouse: Callback[[int, int, int, int, Any | None], None], param: Any | None = ... ) -> None: ... ``` --- .../typing_stubs_generation/api_refinement.py | 89 ++++++++++++++++++- .../typing_stubs_generation/nodes/__init__.py | 3 +- 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/api_refinement.py b/modules/python/src2/typing_stubs_generation/api_refinement.py index c3446e7a45..593ee1639e 100644 --- a/modules/python/src2/typing_stubs_generation/api_refinement.py +++ b/modules/python/src2/typing_stubs_generation/api_refinement.py @@ -6,12 +6,14 @@ from typing import cast, Sequence, Callable, Iterable from .nodes import (NamespaceNode, FunctionNode, OptionalTypeNode, TypeNode, ClassProperty, PrimitiveTypeNode, ASTNodeTypeNode, - AggregatedTypeNode) + AggregatedTypeNode, CallableTypeNode, AnyTypeNode, + TupleTypeNode, UnionTypeNode) from .ast_utils import (find_function_node, SymbolName, for_each_function_overload) def apply_manual_api_refinement(root: NamespaceNode) -> None: + refine_highgui_module(root) refine_cuda_module(root) export_matrix_type_constants(root) # Export OpenCV exception class @@ -105,6 +107,91 @@ def refine_cuda_module(root: NamespaceNode) -> None: fix_namespace_usage_scope(ns) +def refine_highgui_module(root: NamespaceNode) -> None: + # Check if library is built with enabled highgui module + if "destroyAllWindows" not in root.functions: + return + """ + def createTrackbar(trackbarName: str, + windowName: str, + value: int, + count: int, + onChange: Callable[[int], None]) -> None: ... + """ + root.add_function( + "createTrackbar", + [ + FunctionNode.Arg("trackbarName", PrimitiveTypeNode.str_()), + FunctionNode.Arg("windowName", PrimitiveTypeNode.str_()), + FunctionNode.Arg("value", PrimitiveTypeNode.int_()), + FunctionNode.Arg("count", PrimitiveTypeNode.int_()), + FunctionNode.Arg("onChange", + CallableTypeNode("TrackbarCallback", + PrimitiveTypeNode.int_("int"))), + ] + ) + """ + def createButton(buttonName: str, + onChange: Callable[[tuple[int] | tuple[int, Any]], None], + userData: Any | None = ..., + buttonType: int = ..., + initialButtonState: int = ...) -> None: ... + """ + root.add_function( + "createButton", + [ + FunctionNode.Arg("buttonName", PrimitiveTypeNode.str_()), + FunctionNode.Arg( + "onChange", + CallableTypeNode( + "ButtonCallback", + UnionTypeNode( + "onButtonChangeCallbackData", + [ + TupleTypeNode("onButtonChangeCallbackData", + [PrimitiveTypeNode.int_(), ]), + TupleTypeNode("onButtonChangeCallbackData", + [PrimitiveTypeNode.int_(), + AnyTypeNode("void*")]) + ] + ) + )), + FunctionNode.Arg("userData", + OptionalTypeNode(AnyTypeNode("void*")), + default_value="None"), + FunctionNode.Arg("buttonType", PrimitiveTypeNode.int_(), + default_value="0"), + FunctionNode.Arg("initialButtonState", PrimitiveTypeNode.int_(), + default_value="0") + ] + ) + """ + def setMouseCallback( + windowName: str, + onMouse: Callback[[int, int, int, int, Any | None], None], + param: Any | None = ... + ) -> None: ... + """ + root.add_function( + "setMouseCallback", + [ + FunctionNode.Arg("windowName", PrimitiveTypeNode.str_()), + FunctionNode.Arg( + "onMouse", + CallableTypeNode("MouseCallback", [ + PrimitiveTypeNode.int_(), + PrimitiveTypeNode.int_(), + PrimitiveTypeNode.int_(), + PrimitiveTypeNode.int_(), + OptionalTypeNode(AnyTypeNode("void*")) + ]) + ), + FunctionNode.Arg("param", OptionalTypeNode(AnyTypeNode("void*")), + default_value="None") + ] + ) + + def _trim_class_name_from_argument_types( overloads: Iterable[FunctionNode.Overload], class_name: str diff --git a/modules/python/src2/typing_stubs_generation/nodes/__init__.py b/modules/python/src2/typing_stubs_generation/nodes/__init__.py index 82f8df8c92..ad6a92972d 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/__init__.py +++ b/modules/python/src2/typing_stubs_generation/nodes/__init__.py @@ -7,5 +7,6 @@ from .constant_node import ConstantNode from .type_node import ( TypeNode, OptionalTypeNode, UnionTypeNode, NoneTypeNode, TupleTypeNode, ASTNodeTypeNode, AliasTypeNode, SequenceTypeNode, AnyTypeNode, - AggregatedTypeNode, NDArrayTypeNode, AliasRefTypeNode, PrimitiveTypeNode + AggregatedTypeNode, NDArrayTypeNode, AliasRefTypeNode, PrimitiveTypeNode, + CallableTypeNode, ) From 94de7e5d2146b53138a293bb792953748bf6a3b7 Mon Sep 17 00:00:00 2001 From: Vincent Rabaud Date: Wed, 26 Jul 2023 19:00:22 +0200 Subject: [PATCH 101/105] Merge pull request #24042 from vrabaud:circle Fix harmless ASAN error. #24042 For an empty radius, &v[0] would be accessed (though the called functions would not use it due to v.size() being 0). Also add checks for emptyness and fix the first element checks, in case we get INT_MAX to compare to. ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/imgproc/src/drawing.cpp | 5 +++-- modules/imgproc/test/test_drawing.cpp | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/imgproc/src/drawing.cpp b/modules/imgproc/src/drawing.cpp index cbd238aca9..40ba22430f 100644 --- a/modules/imgproc/src/drawing.cpp +++ b/modules/imgproc/src/drawing.cpp @@ -939,6 +939,7 @@ void ellipse2Poly( Point center, Size axes, int angle, } // If there are no points, it's a zero-size polygon + CV_Assert( !pts.empty() ); if (pts.size() == 1) { pts.assign(2, center); } @@ -1001,6 +1002,7 @@ void ellipse2Poly( Point2d center, Size2d axes, int angle, } // If there are no points, it's a zero-size polygon + CV_Assert( !pts.empty() ); if( pts.size() == 1) { pts.assign(2,center); } @@ -1021,7 +1023,6 @@ EllipseEx( Mat& img, Point2l center, Size2l axes, std::vector v; Point2l prevPt(0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFF); - v.resize(0); for (unsigned int i = 0; i < _v.size(); ++i) { Point2l pt; @@ -1036,7 +1037,7 @@ EllipseEx( Mat& img, Point2l center, Size2l axes, } // If there are no points, it's a zero-size polygon - if (v.size() == 1) { + if (v.size() <= 1) { v.assign(2, center); } diff --git a/modules/imgproc/test/test_drawing.cpp b/modules/imgproc/test/test_drawing.cpp index 23208ab881..02095cd6b7 100644 --- a/modules/imgproc/test/test_drawing.cpp +++ b/modules/imgproc/test/test_drawing.cpp @@ -921,4 +921,11 @@ TEST(Drawing, circle_overflow) cv::circle(matrix, cv::Point(275, -2147483318), 2147483647, kBlue, 1, 8, 0); } +TEST(Drawing, circle_memory_access) +{ + cv::Mat1b matrix = cv::Mat1b::zeros(10, 10); + cv::Scalar kBlue = cv::Scalar(0, 0, 255); + cv::circle(matrix, cv::Point(-1, -1), 0, kBlue, 2, 8, 16); +} + }} // namespace From e59b8cd905a07d0bae26f522c1e042ea240c1f31 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Wed, 26 Jul 2023 16:47:26 +0300 Subject: [PATCH 102/105] feat: add typing stub for redirectError Python interface for `redirectError`: ```python def redirectError( onError: Callable[[int, str, str, str, int], None] | None ) -> None: ... ``` --- .../typing_stubs_generation/api_refinement.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/modules/python/src2/typing_stubs_generation/api_refinement.py b/modules/python/src2/typing_stubs_generation/api_refinement.py index 593ee1639e..10aebf8ff2 100644 --- a/modules/python/src2/typing_stubs_generation/api_refinement.py +++ b/modules/python/src2/typing_stubs_generation/api_refinement.py @@ -25,6 +25,29 @@ def apply_manual_api_refinement(root: NamespaceNode) -> None: version_constant = root.add_constant("__version__", "") version_constant._value_type = "str" + """ + def redirectError( + onError: Callable[[int, str, str, str, int], None] | None + ) -> None: ... + """ + root.add_function("redirectError", [ + FunctionNode.Arg( + "onError", + OptionalTypeNode( + CallableTypeNode( + "ErrorCallback", + [ + PrimitiveTypeNode.int_(), + PrimitiveTypeNode.str_(), + PrimitiveTypeNode.str_(), + PrimitiveTypeNode.str_(), + PrimitiveTypeNode.int_() + ] + ) + ) + ) + ]) + def export_matrix_type_constants(root: NamespaceNode) -> None: MAX_PREDEFINED_CHANNELS = 4 From 0c5d74ec1a9a899208ac0703960414f5c835d1a1 Mon Sep 17 00:00:00 2001 From: Vadim Levin Date: Thu, 27 Jul 2023 11:28:00 +0300 Subject: [PATCH 103/105] Merge pull request #24066 from VadimLevin:dev/vlevin/python-typing-register-dnn-layer Python typing refinement for dnn_registerLayer/dnn_unregisterLayer functions #24066 This patch introduces typings generation for `dnn_registerLayer`/`dnn_unregisterLayer` manually defined in [`cv2/modules/dnn/misc/python/pyopencv_dnn.hpp`](https://github.com/opencv/opencv/blob/4.x/modules/dnn/misc/python/pyopencv_dnn.hpp) Updates: - Add `LayerProtocol` to `cv2/dnn/__init__.pyi`: ```python class LayerProtocol(Protocol): def __init__( self, params: dict[str, DictValue], blobs: typing.Sequence[cv2.typing.MatLike] ) -> None: ... def getMemoryShapes( self, inputs: typing.Sequence[typing.Sequence[int]] ) -> typing.Sequence[typing.Sequence[int]]: ... def forward( self, inputs: typing.Sequence[cv2.typing.MatLike] ) -> typing.Sequence[cv2.typing.MatLike]: ... ``` - Add `dnn_registerLayer` function to `cv2/__init__.pyi`: ```python def dnn_registerLayer(layerTypeName: str, layerClass: typing.Type[LayerProtocol]) -> None: ... ``` - Add `dnn_unregisterLayer` function to `cv2/__init__.pyi`: ```python def dnn_unregisterLayer(layerTypeName: str) -> None: ... ``` ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [ ] There is a reference to the original bug report and related work - [ ] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- .../typing_stubs_generation/api_refinement.py | 85 ++++++++++++++++++- .../src2/typing_stubs_generation/ast_utils.py | 14 +-- .../typing_stubs_generation/generation.py | 63 +++++++++----- .../typing_stubs_generation/nodes/__init__.py | 4 +- .../nodes/class_node.py | 21 +++-- .../nodes/constant_node.py | 4 +- .../nodes/enumeration_node.py | 6 +- .../nodes/function_node.py | 4 +- .../nodes/namespace_node.py | 18 ++-- .../typing_stubs_generation/nodes/node.py | 48 +++++------ .../nodes/type_node.py | 15 ++++ 11 files changed, 205 insertions(+), 77 deletions(-) diff --git a/modules/python/src2/typing_stubs_generation/api_refinement.py b/modules/python/src2/typing_stubs_generation/api_refinement.py index 10aebf8ff2..b04bb4a196 100644 --- a/modules/python/src2/typing_stubs_generation/api_refinement.py +++ b/modules/python/src2/typing_stubs_generation/api_refinement.py @@ -7,15 +7,18 @@ from typing import cast, Sequence, Callable, Iterable from .nodes import (NamespaceNode, FunctionNode, OptionalTypeNode, TypeNode, ClassProperty, PrimitiveTypeNode, ASTNodeTypeNode, AggregatedTypeNode, CallableTypeNode, AnyTypeNode, - TupleTypeNode, UnionTypeNode) + TupleTypeNode, UnionTypeNode, ProtocolClassNode, + DictTypeNode, ClassTypeNode) from .ast_utils import (find_function_node, SymbolName, for_each_function_overload) +from .types_conversion import create_type_node def apply_manual_api_refinement(root: NamespaceNode) -> None: refine_highgui_module(root) refine_cuda_module(root) export_matrix_type_constants(root) + refine_dnn_module(root) # Export OpenCV exception class builtin_exception = root.add_class("Exception") builtin_exception.is_exported = False @@ -215,6 +218,86 @@ def refine_highgui_module(root: NamespaceNode) -> None: ) +def refine_dnn_module(root: NamespaceNode) -> None: + if "dnn" not in root.namespaces: + return + dnn_module = root.namespaces["dnn"] + + """ + class LayerProtocol(Protocol): + def __init__( + self, params: dict[str, DictValue], + blobs: typing.Sequence[cv2.typing.MatLike] + ) -> None: ... + + def getMemoryShapes( + self, inputs: typing.Sequence[typing.Sequence[int]] + ) -> typing.Sequence[typing.Sequence[int]]: ... + + def forward( + self, inputs: typing.Sequence[cv2.typing.MatLike] + ) -> typing.Sequence[cv2.typing.MatLike]: ... + """ + layer_proto = ProtocolClassNode("LayerProtocol", dnn_module) + layer_proto.add_function( + "__init__", + arguments=[ + FunctionNode.Arg( + "params", + DictTypeNode( + "LayerParams", PrimitiveTypeNode.str_(), + create_type_node("cv::dnn::DictValue") + ) + ), + FunctionNode.Arg("blobs", create_type_node("vector")) + ] + ) + layer_proto.add_function( + "getMemoryShapes", + arguments=[ + FunctionNode.Arg("inputs", + create_type_node("vector>")) + ], + return_type=FunctionNode.RetType( + create_type_node("vector>") + ) + ) + layer_proto.add_function( + "forward", + arguments=[ + FunctionNode.Arg("inputs", create_type_node("vector")) + ], + return_type=FunctionNode.RetType(create_type_node("vector")) + ) + + """ + def dnn_registerLayer(layerTypeName: str, + layerClass: typing.Type[LayerProtocol]) -> None: ... + """ + root.add_function( + "dnn_registerLayer", + arguments=[ + FunctionNode.Arg("layerTypeName", PrimitiveTypeNode.str_()), + FunctionNode.Arg( + "layerClass", + ClassTypeNode(ASTNodeTypeNode( + layer_proto.export_name, f"dnn.{layer_proto.export_name}" + )) + ) + ] + ) + + """ + def dnn_unregisterLayer(layerTypeName: str) -> None: ... + """ + root.add_function( + "dnn_unregisterLayer", + arguments=[ + FunctionNode.Arg("layerTypeName", PrimitiveTypeNode.str_()) + ] + ) + + def _trim_class_name_from_argument_types( overloads: Iterable[FunctionNode.Overload], class_name: str diff --git a/modules/python/src2/typing_stubs_generation/ast_utils.py b/modules/python/src2/typing_stubs_generation/ast_utils.py index 47c06571a5..e8ef52d19a 100644 --- a/modules/python/src2/typing_stubs_generation/ast_utils.py +++ b/modules/python/src2/typing_stubs_generation/ast_utils.py @@ -1,5 +1,5 @@ from typing import (NamedTuple, Sequence, Tuple, Union, List, - Dict, Callable, Optional, Generator) + Dict, Callable, Optional, Generator, cast) import keyword from .nodes import (ASTNode, NamespaceNode, ClassNode, FunctionNode, @@ -204,9 +204,7 @@ def create_function_node_in_scope(scope: Union[NamespaceNode, ClassNode], outlist = variant.py_outlist for _, argno in outlist: assert argno >= 0, \ - "Logic Error! Outlist contains function return type: {}".format( - outlist - ) + f"Logic Error! Outlist contains function return type: {outlist}" ret_types.append(create_type_node(variant.args[argno].tp)) @@ -379,7 +377,7 @@ def get_enclosing_namespace( node.full_export_name, node.native_name ) if class_node_callback: - class_node_callback(parent_node) + class_node_callback(cast(ClassNode, parent_node)) parent_node = parent_node.parent return parent_node @@ -395,12 +393,14 @@ def get_enum_module_and_export_name(enum_node: EnumerationNode) -> Tuple[str, st Returns: Tuple[str, str]: a pair of enum export name and its full module name. """ + enum_export_name = enum_node.export_name + def update_full_export_name(class_node: ClassNode) -> None: nonlocal enum_export_name enum_export_name = class_node.export_name + "_" + enum_export_name - enum_export_name = enum_node.export_name - namespace_node = get_enclosing_namespace(enum_node, update_full_export_name) + namespace_node = get_enclosing_namespace(enum_node, + update_full_export_name) return enum_export_name, namespace_node.full_export_name diff --git a/modules/python/src2/typing_stubs_generation/generation.py b/modules/python/src2/typing_stubs_generation/generation.py index 41525d4114..c5578bb3a8 100644 --- a/modules/python/src2/typing_stubs_generation/generation.py +++ b/modules/python/src2/typing_stubs_generation/generation.py @@ -3,7 +3,7 @@ __all__ = ("generate_typing_stubs", ) from io import StringIO from pathlib import Path import re -from typing import (Type, Callable, NamedTuple, Union, Set, Dict, +from typing import (Callable, NamedTuple, Union, Set, Dict, Collection, Tuple, List) import warnings @@ -16,7 +16,8 @@ from .predefined_types import PREDEFINED_TYPES from .api_refinement import apply_manual_api_refinement from .nodes import (ASTNode, ASTNodeType, NamespaceNode, ClassNode, - FunctionNode, EnumerationNode, ConstantNode) + FunctionNode, EnumerationNode, ConstantNode, + ProtocolClassNode) from .nodes.type_node import (TypeNode, AliasTypeNode, AliasRefTypeNode, AggregatedTypeNode, ASTNodeTypeNode, @@ -112,11 +113,13 @@ def _generate_typing_stubs(root: NamespaceNode, output_path: Path) -> None: # NOTE: Enumerations require special handling, because all enumeration # constants are exposed as module attributes has_enums = _generate_section_stub( - StubSection("# Enumerations", EnumerationNode), root, output_stream, 0 + StubSection("# Enumerations", ASTNodeType.Enumeration), root, + output_stream, 0 ) # Collect all enums from class level and export them to module level for class_node in root.classes.values(): - if _generate_enums_from_classes_tree(class_node, output_stream, indent=0): + if _generate_enums_from_classes_tree(class_node, output_stream, + indent=0): has_enums = True # 2 empty lines between enum and classes definitions if has_enums: @@ -134,14 +137,15 @@ def _generate_typing_stubs(root: NamespaceNode, output_path: Path) -> None: class StubSection(NamedTuple): name: str - node_type: Type[ASTNode] + node_type: ASTNodeType STUB_SECTIONS = ( - StubSection("# Constants", ConstantNode), - # StubSection("# Enumerations", EnumerationNode), # Skipped for now (special rules) - StubSection("# Classes", ClassNode), - StubSection("# Functions", FunctionNode) + StubSection("# Constants", ASTNodeType.Constant), + # Enumerations are skipped due to special handling rules + # StubSection("# Enumerations", ASTNodeType.Enumeration), + StubSection("# Classes", ASTNodeType.Class), + StubSection("# Functions", ASTNodeType.Function) ) @@ -250,9 +254,9 @@ def _generate_class_stub(class_node: ClassNode, output_stream: StringIO, else: bases.append(base.export_name) - inheritance_str = "({})".format( - ', '.join(bases) - ) + inheritance_str = f"({', '.join(bases)})" + elif isinstance(class_node, ProtocolClassNode): + inheritance_str = "(Protocol)" else: inheritance_str = "" @@ -547,7 +551,8 @@ def check_overload_presence(node: Union[NamespaceNode, ClassNode]) -> bool: return True return False -def _collect_required_imports(root: NamespaceNode) -> Set[str]: + +def _collect_required_imports(root: NamespaceNode) -> Collection[str]: """Collects all imports required for classes and functions typing stubs declarations. @@ -555,8 +560,8 @@ def _collect_required_imports(root: NamespaceNode) -> Set[str]: root (NamespaceNode): Namespace node to collect imports for Returns: - Set[str]: Collection of unique `import smth` statements required for - classes and function declarations of `root` node. + Collection[str]: Collection of unique `import smth` statements required + for classes and function declarations of `root` node. """ def _add_required_usage_imports(type_node: TypeNode, imports: Set[str]): @@ -569,6 +574,7 @@ def _collect_required_imports(root: NamespaceNode) -> Set[str]: has_overload = check_overload_presence(root) # if there is no module-level functions with overload, check its presence # during class traversing, including their inner-classes + has_protocol = False for cls in for_each_class(root): if not has_overload and check_overload_presence(cls): has_overload = True @@ -583,6 +589,8 @@ def _collect_required_imports(root: NamespaceNode) -> Set[str]: required_imports.add( "import " + base_namespace.full_export_name ) + if isinstance(cls, ProtocolClassNode): + has_protocol = True if has_overload: required_imports.add("import typing") @@ -599,7 +607,20 @@ def _collect_required_imports(root: NamespaceNode) -> Set[str]: if root_import in required_imports: required_imports.remove(root_import) - return required_imports + if has_protocol: + required_imports.add("import sys") + ordered_required_imports = sorted(required_imports) + + # Protocol import always goes as last import statement + if has_protocol: + ordered_required_imports.append( + """if sys.version_info >= (3, 8): + from typing import Protocol +else: + from typing_extensions import Protocol""" + ) + + return ordered_required_imports def _populate_reexported_symbols(root: NamespaceNode) -> None: @@ -666,7 +687,7 @@ def _write_required_imports(required_imports: Collection[str], output_stream (StringIO): Output stream for import statements. """ - for required_import in sorted(required_imports): + for required_import in required_imports: output_stream.write(required_import) output_stream.write("\n") if len(required_imports): @@ -803,8 +824,8 @@ StubGenerator = Callable[[ASTNode, StringIO, int], None] NODE_TYPE_TO_STUB_GENERATOR = { - ClassNode: _generate_class_stub, - ConstantNode: _generate_constant_stub, - EnumerationNode: _generate_enumeration_stub, - FunctionNode: _generate_function_stub + ASTNodeType.Class: _generate_class_stub, + ASTNodeType.Constant: _generate_constant_stub, + ASTNodeType.Enumeration: _generate_enumeration_stub, + ASTNodeType.Function: _generate_function_stub } diff --git a/modules/python/src2/typing_stubs_generation/nodes/__init__.py b/modules/python/src2/typing_stubs_generation/nodes/__init__.py index ad6a92972d..a2dbc49964 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/__init__.py +++ b/modules/python/src2/typing_stubs_generation/nodes/__init__.py @@ -1,6 +1,6 @@ from .node import ASTNode, ASTNodeType from .namespace_node import NamespaceNode -from .class_node import ClassNode, ClassProperty +from .class_node import ClassNode, ClassProperty, ProtocolClassNode from .function_node import FunctionNode from .enumeration_node import EnumerationNode from .constant_node import ConstantNode @@ -8,5 +8,5 @@ from .type_node import ( TypeNode, OptionalTypeNode, UnionTypeNode, NoneTypeNode, TupleTypeNode, ASTNodeTypeNode, AliasTypeNode, SequenceTypeNode, AnyTypeNode, AggregatedTypeNode, NDArrayTypeNode, AliasRefTypeNode, PrimitiveTypeNode, - CallableTypeNode, + CallableTypeNode, DictTypeNode, ClassTypeNode ) diff --git a/modules/python/src2/typing_stubs_generation/nodes/class_node.py b/modules/python/src2/typing_stubs_generation/nodes/class_node.py index b3d394786f..78bd01d61f 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/class_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/class_node.py @@ -63,8 +63,9 @@ class ClassNode(ASTNode): return 1 + sum(base.weight for base in self.bases) @property - def children_types(self) -> Tuple[Type[ASTNode], ...]: - return (ClassNode, FunctionNode, EnumerationNode, ConstantNode) + def children_types(self) -> Tuple[ASTNodeType, ...]: + return (ASTNodeType.Class, ASTNodeType.Function, + ASTNodeType.Enumeration, ASTNodeType.Constant) @property def node_type(self) -> ASTNodeType: @@ -72,19 +73,19 @@ class ClassNode(ASTNode): @property def classes(self) -> Dict[str, "ClassNode"]: - return self._children[ClassNode] + return self._children[ASTNodeType.Class] @property def functions(self) -> Dict[str, FunctionNode]: - return self._children[FunctionNode] + return self._children[ASTNodeType.Function] @property def enumerations(self) -> Dict[str, EnumerationNode]: - return self._children[EnumerationNode] + return self._children[ASTNodeType.Enumeration] @property def constants(self) -> Dict[str, ConstantNode]: - return self._children[ConstantNode] + return self._children[ASTNodeType.Constant] def add_class(self, name: str, bases: Sequence["weakref.ProxyType[ClassNode]"] = (), @@ -179,3 +180,11 @@ class ClassNode(ASTNode): self.full_export_name, root.full_export_name, errors ) ) + + +class ProtocolClassNode(ClassNode): + def __init__(self, name: str, parent: Optional[ASTNode] = None, + export_name: Optional[str] = None, + properties: Sequence[ClassProperty] = ()) -> None: + super().__init__(name, parent, export_name, bases=(), + properties=properties) diff --git a/modules/python/src2/typing_stubs_generation/nodes/constant_node.py b/modules/python/src2/typing_stubs_generation/nodes/constant_node.py index 3dae255a3c..ad18b80b50 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/constant_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/constant_node.py @@ -1,4 +1,4 @@ -from typing import Type, Optional, Tuple +from typing import Optional, Tuple from .node import ASTNode, ASTNodeType @@ -14,7 +14,7 @@ class ConstantNode(ASTNode): self._value_type = "int" @property - def children_types(self) -> Tuple[Type[ASTNode], ...]: + def children_types(self) -> Tuple[ASTNodeType, ...]: return () @property diff --git a/modules/python/src2/typing_stubs_generation/nodes/enumeration_node.py b/modules/python/src2/typing_stubs_generation/nodes/enumeration_node.py index 249f29db1c..12b996d1a1 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/enumeration_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/enumeration_node.py @@ -18,8 +18,8 @@ class EnumerationNode(ASTNode): self.is_scoped = is_scoped @property - def children_types(self) -> Tuple[Type[ASTNode], ...]: - return (ConstantNode, ) + def children_types(self) -> Tuple[ASTNodeType, ...]: + return (ASTNodeType.Constant, ) @property def node_type(self) -> ASTNodeType: @@ -27,7 +27,7 @@ class EnumerationNode(ASTNode): @property def constants(self) -> Dict[str, ConstantNode]: - return self._children[ConstantNode] + return self._children[ASTNodeType.Constant] def add_constant(self, name: str, value: str) -> ConstantNode: return self._add_child(ConstantNode, name, value=value) diff --git a/modules/python/src2/typing_stubs_generation/nodes/function_node.py b/modules/python/src2/typing_stubs_generation/nodes/function_node.py index 4ebb90633e..568676f69a 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/function_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/function_node.py @@ -1,4 +1,4 @@ -from typing import NamedTuple, Sequence, Type, Optional, Tuple, List +from typing import NamedTuple, Sequence, Optional, Tuple, List from .node import ASTNode, ASTNodeType from .type_node import TypeNode, NoneTypeNode, TypeResolutionError @@ -98,7 +98,7 @@ class FunctionNode(ASTNode): return ASTNodeType.Function @property - def children_types(self) -> Tuple[Type[ASTNode], ...]: + def children_types(self) -> Tuple[ASTNodeType, ...]: return () def add_overload(self, arguments: Sequence["FunctionNode.Arg"] = (), diff --git a/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py b/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py index 8d0d04b5a5..445ef77b7d 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/namespace_node.py @@ -1,7 +1,7 @@ import itertools import weakref from collections import defaultdict -from typing import Dict, List, Optional, Sequence, Tuple, Type +from typing import Dict, List, Optional, Sequence, Tuple from .class_node import ClassNode, ClassProperty from .constant_node import ConstantNode @@ -33,29 +33,29 @@ class NamespaceNode(ASTNode): return ASTNodeType.Namespace @property - def children_types(self) -> Tuple[Type[ASTNode], ...]: - return (NamespaceNode, ClassNode, FunctionNode, - EnumerationNode, ConstantNode) + def children_types(self) -> Tuple[ASTNodeType, ...]: + return (ASTNodeType.Namespace, ASTNodeType.Class, ASTNodeType.Function, + ASTNodeType.Enumeration, ASTNodeType.Constant) @property def namespaces(self) -> Dict[str, "NamespaceNode"]: - return self._children[NamespaceNode] + return self._children[ASTNodeType.Namespace] @property def classes(self) -> Dict[str, ClassNode]: - return self._children[ClassNode] + return self._children[ASTNodeType.Class] @property def functions(self) -> Dict[str, FunctionNode]: - return self._children[FunctionNode] + return self._children[ASTNodeType.Function] @property def enumerations(self) -> Dict[str, EnumerationNode]: - return self._children[EnumerationNode] + return self._children[ASTNodeType.Enumeration] @property def constants(self) -> Dict[str, ConstantNode]: - return self._children[ConstantNode] + return self._children[ASTNodeType.Constant] def add_namespace(self, name: str) -> "NamespaceNode": return self._add_child(NamespaceNode, name) diff --git a/modules/python/src2/typing_stubs_generation/nodes/node.py b/modules/python/src2/typing_stubs_generation/nodes/node.py index 126f668018..20efb47145 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/node.py @@ -70,22 +70,22 @@ class ASTNode: self._parent: Optional["ASTNode"] = None self.parent = parent self.is_exported = True - self._children: DefaultDict[NodeType, NameToNode] = defaultdict(dict) + self._children: DefaultDict[ASTNodeType, NameToNode] = defaultdict(dict) def __str__(self) -> str: return "{}('{}' exported as '{}')".format( - type(self).__name__.replace("Node", ""), self.name, self.export_name + self.node_type.name, self.name, self.export_name ) def __repr__(self) -> str: return str(self) @abc.abstractproperty - def children_types(self) -> Tuple[Type["ASTNode"], ...]: + def children_types(self) -> Tuple[ASTNodeType, ...]: """Set of ASTNode types that are allowed to be children of this node Returns: - Tuple[Type[ASTNode], ...]: Types of children nodes + Tuple[ASTNodeType, ...]: Types of children nodes """ pass @@ -99,6 +99,9 @@ class ASTNode: """ pass + def node_type_name(self) -> str: + return f"{self.node_type.name}::{self.name}" + @property def name(self) -> str: return self.__name @@ -126,11 +129,11 @@ class ASTNode: "but got: {}".format(type(value)) if value is not None: - value.__check_child_before_add(type(self), self.name) + value.__check_child_before_add(self, self.name) # Detach from previous parent if self._parent is not None: - self._parent._children[type(self)].pop(self.name) + self._parent._children[self.node_type].pop(self.name) if value is None: self._parent = None @@ -138,28 +141,26 @@ class ASTNode: # Set a weak reference to a new parent and add self to its children self._parent = weakref.proxy(value) - value._children[type(self)][self.name] = self + value._children[self.node_type][self.name] = self - def __check_child_before_add(self, child_type: Type[ASTNodeSubtype], + def __check_child_before_add(self, child: ASTNodeSubtype, name: str) -> None: - assert len(self.children_types) > 0, \ - "Trying to add child node '{}::{}' to node '{}::{}' " \ - "that can't have children nodes".format(child_type.__name__, name, - type(self).__name__, - self.name) + assert len(self.children_types) > 0, ( + f"Trying to add child node '{child.node_type_name}' to node " + f"'{self.node_type_name}' that can't have children nodes" + ) - assert child_type in self.children_types, \ - "Trying to add child node '{}::{}' to node '{}::{}' " \ + assert child.node_type in self.children_types, \ + "Trying to add child node '{}' to node '{}' " \ "that supports only ({}) as its children types".format( - child_type.__name__, name, type(self).__name__, self.name, - ",".join(t.__name__ for t in self.children_types) + child.node_type_name, self.node_type_name, + ",".join(t.name for t in self.children_types) ) - if self._find_child(child_type, name) is not None: + if self._find_child(child.node_type, name) is not None: raise ValueError( - "Node '{}::{}' already has a child '{}::{}'".format( - type(self).__name__, self.name, child_type.__name__, name - ) + f"Node '{self.node_type_name}' already has a " + f"child '{child.node_type_name}'" ) def _add_child(self, child_type: Type[ASTNodeSubtype], name: str, @@ -180,15 +181,14 @@ class ASTNode: Returns: ASTNodeSubtype: Created ASTNode """ - self.__check_child_before_add(child_type, name) return child_type(name, parent=self, **kwargs) - def _find_child(self, child_type: Type[ASTNodeSubtype], + def _find_child(self, child_type: ASTNodeType, name: str) -> Optional[ASTNodeSubtype]: """Looks for child node with the given type and name. Args: - child_type (Type[ASTNodeSubtype]): Type of the child node. + child_type (ASTNodeType): Type of the child node. name (str): Name of the child node. Returns: diff --git a/modules/python/src2/typing_stubs_generation/nodes/type_node.py b/modules/python/src2/typing_stubs_generation/nodes/type_node.py index 8af4dd0039..089ff2ee9d 100644 --- a/modules/python/src2/typing_stubs_generation/nodes/type_node.py +++ b/modules/python/src2/typing_stubs_generation/nodes/type_node.py @@ -839,6 +839,21 @@ class CallableTypeNode(AggregatedTypeNode): yield from super().required_usage_imports +class ClassTypeNode(ContainerTypeNode): + """Type node representing types themselves (refer to typing.Type) + """ + def __init__(self, value: TypeNode) -> None: + super().__init__(value.ctype_name, (value,)) + + @property + def type_format(self) -> str: + return "typing.Type[{}]" + + @property + def types_separator(self) -> str: + return ", " + + def _resolve_symbol(root: Optional[ASTNode], full_symbol_name: str) -> Optional[ASTNode]: """Searches for a symbol with the given full export name in the AST starting from the `root`. From ab6bffc6f89ae65b5691ebc7931030a8cea62cf2 Mon Sep 17 00:00:00 2001 From: SaltFish-T <47022023+SaltFish-T@users.noreply.github.com> Date: Thu, 27 Jul 2023 19:21:30 +0800 Subject: [PATCH 104/105] Merge pull request #23936 from SaltFish-T:4.x Update opencv dnn to support cann version >=6.3 #23936 1.modify the search path of "libopsproto.so" in OpenCVFindCANN.cmake 2.add the search path of "libgraph_base.so" in OpenCVFindCANN.cmake 3.automatic check Ascend socVersion,and test on Ascend310/Ascend310B/Ascend910B well --- cmake/OpenCVFindCANN.cmake | 45 ++++++++++++++++++++++++++++++------ modules/dnn/src/net_cann.cpp | 4 ++-- modules/dnn/src/op_cann.hpp | 6 ++++- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/cmake/OpenCVFindCANN.cmake b/cmake/OpenCVFindCANN.cmake index b0b8e35c6b..e1cd054a37 100644 --- a/cmake/OpenCVFindCANN.cmake +++ b/cmake/OpenCVFindCANN.cmake @@ -5,13 +5,24 @@ if("cann${CANN_INSTALL_DIR}" STREQUAL "cann" AND DEFINED ENV{ASCEND_TOOLKIT_HOME message(STATUS "CANN: updated CANN_INSTALL_DIR from ASCEND_TOOLKIT_HOME=$ENV{ASCEND_TOOLKIT_HOME}") endif() +if(EXISTS "${CANN_INSTALL_DIR}/opp/op_proto/built-in/inc") + set(CANN_VERSION_BELOW_6_3_ALPHA002 "YES" ) + add_definitions(-DCANN_VERSION_BELOW_6_3_ALPHA002="YES") +endif() + if(CANN_INSTALL_DIR) + # Supported system: UNIX + if(NOT UNIX) + set(HAVE_CANN OFF) + message(WARNING "CANN: CANN toolkit supports unix but not ${CMAKE_SYSTEM_NAME}. Turning off HAVE_CANN") + return() + endif() # Supported platforms: x86-64, arm64 if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64") else() set(HAVE_CANN OFF) - message(STATUS "CANN: CANN toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}. Turning off HAVE_CANN") + message(WARNING "CANN: CANN toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}. Turning off HAVE_CANN") return() endif() @@ -31,7 +42,7 @@ if(CANN_INSTALL_DIR) set(lib_ascendcl ${found_lib_ascendcl}) message(STATUS "CANN: libascendcl.so is found at ${lib_ascendcl}") else() - message(STATUS "CANN: Missing libascendcl.so. Turning off HAVE_CANN") + message(WARNING "CANN: Missing libascendcl.so. Turning off HAVE_CANN") set(HAVE_CANN OFF) return() endif() @@ -42,7 +53,7 @@ if(CANN_INSTALL_DIR) set(lib_graph ${found_lib_graph}) message(STATUS "CANN: libgraph.so is found at ${lib_graph}") else() - message(STATUS "CANN: Missing libgraph.so. Turning off HAVE_CANN") + message(WARNING "CANN: Missing libgraph.so. Turning off HAVE_CANN") set(HAVE_CANN OFF) return() endif() @@ -53,29 +64,49 @@ if(CANN_INSTALL_DIR) set(lib_ge_compiler ${found_lib_ge_compiler}) message(STATUS "CANN: libge_compiler.so is found at ${lib_ge_compiler}") else() - message(STATUS "CANN: Missing libge_compiler.so. Turning off HAVE_CANN") + message(WARNING "CANN: Missing libge_compiler.so. Turning off HAVE_CANN") set(HAVE_CANN OFF) return() endif() # * libopsproto.so - set(lib_opsproto "${CANN_INSTALL_DIR}/opp/op_proto/built-in") + if (CANN_VERSION_BELOW_6_3_ALPHA002) + set(lib_opsproto "${CANN_INSTALL_DIR}/opp/op_proto/built-in/") + else() + if(EXISTS "${CANN_INSTALL_DIR}/opp/built-in/op_proto/lib/linux") + set(lib_opsproto "${CANN_INSTALL_DIR}/opp/built-in/op_proto/lib/linux/${CMAKE_HOST_SYSTEM_PROCESSOR}") + else() + set(lib_opsproto "${CANN_INSTALL_DIR}/opp/built-in/op_proto") + endif() + endif() find_library(found_lib_opsproto NAMES opsproto PATHS ${lib_opsproto} NO_DEFAULT_PATH) if(found_lib_opsproto) set(lib_opsproto ${found_lib_opsproto}) message(STATUS "CANN: libopsproto.so is found at ${lib_opsproto}") else() - message(STATUS "CANN: Missing libopsproto.so. Turning off HAVE_CANN") + message(WARNING "CANN: Missing libopsproto.so can't found at ${lib_opsproto}. Turning off HAVE_CANN") set(HAVE_CANN OFF) return() endif() - set(libs_cann "") list(APPEND libs_cann ${lib_ascendcl}) list(APPEND libs_cann ${lib_opsproto}) list(APPEND libs_cann ${lib_graph}) list(APPEND libs_cann ${lib_ge_compiler}) + # * lib_graph_base.so + if(NOT CANN_VERSION_BELOW_6_3_ALPHA002) + set(lib_graph_base "${CANN_INSTALL_DIR}/compiler/lib64") + find_library(found_libgraph_base NAMES graph_base PATHS ${lib_graph_base} NO_DEFAULT_PATH) + if(found_libgraph_base) + set(lib_graph_base ${found_libgraph_base}) + message(STATUS "CANN: lib_graph_base.so is found at ${lib_graph_base}") + list(APPEND libs_cann ${lib_graph_base}) + else() + message(STATUS "CANN: Missing lib_graph_base.so. It is only required after cann version 6.3.RC1.alpha002") + endif() + endif() + try_compile(VALID_ASCENDCL "${OpenCV_BINARY_DIR}" "${OpenCV_SOURCE_DIR}/cmake/checks/cann.cpp" diff --git a/modules/dnn/src/net_cann.cpp b/modules/dnn/src/net_cann.cpp index a3eb52200f..103c7c8dd2 100644 --- a/modules/dnn/src/net_cann.cpp +++ b/modules/dnn/src/net_cann.cpp @@ -304,9 +304,9 @@ std::shared_ptr compileCannGraph(std::shared_ptr bool ok; if ((child=fork()) == 0) { - // initialize engine + // initialize engine Ascend310/Ascend310P3/Ascend910B/Ascend310B std::map options = { - {ge::AscendString(ge::ir_option::SOC_VERSION), ge::AscendString("Ascend310")}, + {ge::AscendString(ge::ir_option::SOC_VERSION), ge::AscendString(aclrtGetSocName())}, }; ACL_CHECK_GRAPH_RET(ge::aclgrphBuildInitialize(options)); diff --git a/modules/dnn/src/op_cann.hpp b/modules/dnn/src/op_cann.hpp index c60c311b7f..1d15eab6a3 100644 --- a/modules/dnn/src/op_cann.hpp +++ b/modules/dnn/src/op_cann.hpp @@ -10,7 +10,11 @@ #include "graph/graph.h" // ge::Graph; ge::Operator from operator.h #include "graph/ge_error_codes.h" // GRAPH_SUCCESS, ... -#include "op_proto/built-in/inc/all_ops.h" // ge::Conv2D, ... +#ifdef CANN_VERSION_BELOW_6_3_ALPHA002 + #include "op_proto/built-in/inc/all_ops.h" // ge::Conv2D, ... +#else + #include "built-in/op_proto/inc/all_ops.h" // ge::Conv2D, ... +#endif #include "graph/tensor.h" // ge::Shape, ge::Tensor, ge::TensorDesc #include "graph/types.h" // DT_FLOAT, ... ; FORMAT_NCHW, ... From 677a28fd2a63234c19d9a9f3e0a99b8645e99696 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Thu, 27 Jul 2023 16:36:40 +0300 Subject: [PATCH 105/105] Merge pull request #24056 from dkurt:eltwise_prelu PReLU with element-wise scales #24056 ### Pull Request Readiness Checklist resolves https://github.com/opencv/opencv/issues/24051 See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/layers/elementwise_layers.cpp | 110 ++++++++++++++++-- modules/dnn/src/opencl/activations.cl | 15 ++- modules/dnn/src/tensorflow/tf_importer.cpp | 23 ++++ modules/dnn/test/test_tf_importer.cpp | 8 +- 4 files changed, 141 insertions(+), 15 deletions(-) diff --git a/modules/dnn/src/layers/elementwise_layers.cpp b/modules/dnn/src/layers/elementwise_layers.cpp index 4c2e0f3b07..2a34b9400b 100644 --- a/modules/dnn/src/layers/elementwise_layers.cpp +++ b/modules/dnn/src/layers/elementwise_layers.cpp @@ -138,7 +138,7 @@ public: { const float* srcptr = src_->ptr(i) + stripeStart; float* dstptr = dst_->ptr(i) + stripeStart; - func_->apply(srcptr, dstptr, (int)(stripeEnd - stripeStart), planeSize, 0, outCn); + func_->apply(srcptr, dstptr, stripeStart, (int)(stripeEnd - stripeStart), planeSize, 0, outCn); } } }; @@ -268,7 +268,7 @@ public: void forwardSlice(const float* src, float* dst, int len, size_t planeSize, int cn0, int cn1) const CV_OVERRIDE { - func.apply(src, dst, len, planeSize, cn0, cn1); + func.apply(src, dst, -1, len, planeSize, cn0, cn1); } #ifdef HAVE_CUDA @@ -355,8 +355,9 @@ struct ReLUFunctor : public BaseFunctor backendId == DNN_BACKEND_CANN; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + void apply(const float* srcptr, float* dstptr, int stripeStart, int len, size_t planeSize, int cn0, int cn1) const { + CV_UNUSED(stripeStart); float s = slope; for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) { @@ -559,8 +560,9 @@ struct ReLU6Functor : public BaseFunctor backendId == DNN_BACKEND_CANN; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + void apply(const float* srcptr, float* dstptr, int stripeStart, int len, size_t planeSize, int cn0, int cn1) const { + CV_UNUSED(stripeStart); for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) { int i = 0; @@ -704,8 +706,9 @@ struct ReLU6Functor : public BaseFunctor template struct BaseDefaultFunctor : public BaseFunctor { - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + void apply(const float* srcptr, float* dstptr, int stripeStart, int len, size_t planeSize, int cn0, int cn1) const { + CV_UNUSED(stripeStart); for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize ) { for( int i = 0; i < len; i++ ) @@ -2226,8 +2229,9 @@ struct PowerFunctor : public BaseFunctor shift = originShift; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + void apply(const float* srcptr, float* dstptr, int stripeStart, int len, size_t planeSize, int cn0, int cn1) const { + CV_UNUSED(stripeStart); float a = scale, b = shift, p = power; if( p == 1.f ) { @@ -2452,6 +2456,7 @@ struct ChannelsPReLUFunctor : public BaseFunctor Mat scale; #ifdef HAVE_OPENCL UMat scale_umat; + std::string oclKernelName = "ChannelsPReLUForward"; #endif explicit ChannelsPReLUFunctor(const Mat& scale_=Mat()) : scale(scale_) @@ -2470,8 +2475,9 @@ struct ChannelsPReLUFunctor : public BaseFunctor backendId == DNN_BACKEND_CANN; } - void apply(const float* srcptr, float* dstptr, int len, size_t planeSize, int cn0, int cn1) const + void apply(const float* srcptr, float* dstptr, int stripeStart, int len, size_t planeSize, int cn0, int cn1) const { + CV_UNUSED(stripeStart); CV_Assert(scale.isContinuous() && scale.type() == CV_32F); const float* scaleptr = scale.ptr(); @@ -2525,7 +2531,7 @@ struct ChannelsPReLUFunctor : public BaseFunctor UMat& src = inputs[i]; UMat& dst = outputs[i]; - ocl::Kernel kernel("PReLUForward", ocl::dnn::activations_oclsrc, buildopt); + ocl::Kernel kernel(oclKernelName.c_str(), ocl::dnn::activations_oclsrc, buildopt); kernel.set(0, (int)src.total()); kernel.set(1, (int)src.size[1]); kernel.set(2, (int)total(shape(src), 2)); @@ -2605,6 +2611,75 @@ struct ChannelsPReLUFunctor : public BaseFunctor int64 getFLOPSPerElement() const { return 1; } }; +struct PReLUFunctor : public ChannelsPReLUFunctor +{ + explicit PReLUFunctor(const Mat& scale_=Mat()) : ChannelsPReLUFunctor(scale_) + { +#ifdef HAVE_OPENCL + oclKernelName = "PReLUForward"; +#endif + } + + bool supportBackend(int backendId, int) + { + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_CANN || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; + } + + void apply(const float* srcptr, float* dstptr, int stripeStart, int len, size_t planeSize, int cn0, int cn1) const + { + CV_UNUSED(stripeStart); + CV_Assert(scale.isContinuous() && scale.type() == CV_32F); + + if (stripeStart < 0) + CV_Error(Error::StsNotImplemented, "PReLUFunctor requires stripe offset parameter"); + + const float* scaleptr = scale.ptr() + cn0 * planeSize + stripeStart; + for( int cn = cn0; cn < cn1; cn++, srcptr += planeSize, dstptr += planeSize, scaleptr += planeSize ) + { + int i = 0; + #if CV_SIMD128 + v_float32x4 z = v_setzero_f32(); + for( ; i <= len - 16; i += 16 ) + { + v_float32x4 x0 = v_load(srcptr + i); + v_float32x4 x1 = v_load(srcptr + i + 4); + v_float32x4 x2 = v_load(srcptr + i + 8); + v_float32x4 x3 = v_load(srcptr + i + 12); + v_float32x4 s0 = v_load(scaleptr + i); + v_float32x4 s1 = v_load(scaleptr + i + 4); + v_float32x4 s2 = v_load(scaleptr + i + 8); + v_float32x4 s3 = v_load(scaleptr + i + 12); + x0 = v_select(x0 >= z, x0, x0*s0); + x1 = v_select(x1 >= z, x1, x1*s1); + x2 = v_select(x2 >= z, x2, x2*s2); + x3 = v_select(x3 >= z, x3, x3*s3); + v_store(dstptr + i, x0); + v_store(dstptr + i + 4, x1); + v_store(dstptr + i + 8, x2); + v_store(dstptr + i + 12, x3); + } + #endif + for( ; i < len; i++ ) + { + float x = srcptr[i]; + float s = scaleptr[i]; + dstptr[i] = x >= 0.f ? x : s*x; + } + } + } + +#ifdef HAVE_DNN_NGRAPH + std::shared_ptr initNgraphAPI(const std::shared_ptr& node) + { + auto shape = getShape(scale); + auto slope = std::make_shared(ngraph::element::f32, shape, scale.ptr()); + return std::make_shared(node, slope); + } +#endif // HAVE_DNN_NGRAPH +}; + struct SignFunctor : public BaseDefaultFunctor { typedef SignLayer Layer; @@ -3040,13 +3115,26 @@ Ptr ExpLayer::create(const LayerParams& params) Ptr ChannelsPReLULayer::create(const LayerParams& params) { CV_Assert(params.blobs.size() == 1); - if (params.blobs[0].total() == 1) + Mat scale = params.blobs[0]; + float slope = *scale.ptr(); + if (scale.total() == 1 || countNonZero(scale != slope) == 0) { LayerParams reluParams = params; - reluParams.set("negative_slope", *params.blobs[0].ptr()); + reluParams.set("negative_slope", slope); return ReLULayer::create(reluParams); } - Ptr l(new ElementWiseLayer(ChannelsPReLUFunctor(params.blobs[0]))); + + Ptr l; + // Check first two dimensions of scale (batch, channels) + MatShape scaleShape = shape(scale); + if (std::count_if(scaleShape.begin(), scaleShape.end(), [](int d){ return d != 1;}) > 1) + { + l = new ElementWiseLayer(PReLUFunctor(scale)); + } + else + { + l = new ElementWiseLayer(ChannelsPReLUFunctor(scale)); + } l->setParamsFrom(params); return l; diff --git a/modules/dnn/src/opencl/activations.cl b/modules/dnn/src/opencl/activations.cl index 317d2c1e62..96b56725fb 100644 --- a/modules/dnn/src/opencl/activations.cl +++ b/modules/dnn/src/opencl/activations.cl @@ -73,14 +73,23 @@ __kernel void ReLU6Forward(const int count, __global const T* in, __global T* ou } } +__kernel void ChannelsPReLUForward(const int count, const int channels, const int plane_size, + __global const T* in, __global T* out, + __global const KERNEL_ARG_DTYPE* slope_data) +{ + int index = get_global_id(0); + int c = (index / plane_size) % channels; + if(index < count) + out[index] = in[index] > 0 ? in[index] : in[index] * slope_data[c]; +} + __kernel void PReLUForward(const int count, const int channels, const int plane_size, __global const T* in, __global T* out, __global const KERNEL_ARG_DTYPE* slope_data) { int index = get_global_id(0); - int c = (index / plane_size) % channels; if(index < count) - out[index] = in[index] > 0 ? in[index] : in[index] * slope_data[c]; + out[index] = in[index] > 0 ? in[index] : in[index] * slope_data[index]; } __kernel void TanHForward(const int count, __global T* in, __global T* out) { @@ -352,4 +361,4 @@ __kernel void ReciprocalForward(const int n, __global T* in, __global T* out) int index = get_global_id(0); if(index < n) out[index] = 1.0f/in[index]; -} \ No newline at end of file +} diff --git a/modules/dnn/src/tensorflow/tf_importer.cpp b/modules/dnn/src/tensorflow/tf_importer.cpp index 8e9970528a..a23fac187b 100644 --- a/modules/dnn/src/tensorflow/tf_importer.cpp +++ b/modules/dnn/src/tensorflow/tf_importer.cpp @@ -589,6 +589,7 @@ private: void parsePack (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseClipByValue (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseLeakyRelu (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); + void parsePReLU (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseActivation (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseExpandDims (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); void parseSquare (tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams); @@ -668,6 +669,7 @@ TFImporter::DispatchMap TFImporter::buildDispatchMap() dispatch["Pack"] = &TFImporter::parsePack; dispatch["ClipByValue"] = &TFImporter::parseClipByValue; dispatch["LeakyRelu"] = &TFImporter::parseLeakyRelu; + dispatch["PReLU"] = &TFImporter::parsePReLU; dispatch["Abs"] = dispatch["Tanh"] = dispatch["Sigmoid"] = dispatch["Relu"] = dispatch["Elu"] = dispatch["Exp"] = dispatch["Identity"] = dispatch["Relu6"] = &TFImporter::parseActivation; dispatch["ExpandDims"] = &TFImporter::parseExpandDims; @@ -2622,6 +2624,27 @@ void TFImporter::parseLeakyRelu(tensorflow::GraphDef& net, const tensorflow::Nod connectToAllBlobs(layer_id, dstNet, parsePin(layer.input(0)), id, num_inputs); } +void TFImporter::parsePReLU(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) +{ + const std::string& name = layer.name(); + + Mat scales; + blobFromTensor(getConstBlob(layer, value_id, 1), scales); + + layerParams.blobs.resize(1); + + if (scales.dims == 3) { + // Considering scales from Keras wih HWC layout; + transposeND(scales, {2, 0, 1}, layerParams.blobs[0]); + } else { + layerParams.blobs[0] = scales; + } + + int id = dstNet.addLayer(name, "PReLU", layerParams); + layer_id[name] = id; + connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0); +} + // "Abs" "Tanh" "Sigmoid" "Relu" "Elu" "Exp" "Identity" "Relu6" void TFImporter::parseActivation(tensorflow::GraphDef& net, const tensorflow::NodeDef& layer, LayerParams& layerParams) { diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index b795076f55..e2dfbc706e 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -1675,6 +1675,7 @@ TEST_P(Test_TensorFlow_layers, clip_by_value) TEST_P(Test_TensorFlow_layers, tf2_prelu) { + double l1 = 0, lInf = 0; if (backend == DNN_BACKEND_CUDA) applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); // not supported; only across channels is supported #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000) @@ -1686,6 +1687,11 @@ TEST_P(Test_TensorFlow_layers, tf2_prelu) applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION ); +#elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2023000000) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL) { + l1 = 1e-4; + lInf = 1e-3; + } #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000) if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) { @@ -1705,7 +1711,7 @@ TEST_P(Test_TensorFlow_layers, tf2_prelu) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); #endif - runTensorFlowNet("tf2_prelu"); + runTensorFlowNet("tf2_prelu", false, l1, lInf); } TEST_P(Test_TensorFlow_layers, tf2_permute_nhwc_ncwh)